github/mage
1
0
Fork 0
mirror of https://github.com/correl/mage.git synced 2025-04-10 17:00:08 -09:00
Code Issues Projects Releases Packages Wiki Activity

Concurrency perfomance boost

Browse source 
Parallel execution. Speed increased from 50% (avg in all methods) to 500% (checkForNewCards method)
This commit is contained in:
vraskulin 2016-12-22 18:36:40 +03:00
parent c2ef2b1f47
commit 4f81babdfa
1 changed files with 66 additions and 61 deletions

127
Mage.Client/src/main/java/org/mage/plugins/card/images/DownloadPictures.java
View file

@ -22,6 +22,7 @@ import java.nio.file.AccessDeniedException;
import java.util.*; import java.util.*;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.imageio.IIOImage; import javax.imageio.IIOImage;
import javax.imageio.ImageIO; import javax.imageio.ImageIO;
import javax.imageio.ImageWriteParam; import javax.imageio.ImageWriteParam;
@ -235,18 +236,20 @@ public class DownloadPictures extends DefaultBoundedRangeModel implements Runnab
} }
public static boolean checkForNewCards(List<CardInfo> allCards) { public static boolean checkForNewCards(List<CardInfo> allCards) {
TFile file; AtomicBoolean missedCardTFiles = new AtomicBoolean();
for (CardInfo card : allCards) { allCards.parallelStream().forEach(card -> {
if (!card.getCardNumber().isEmpty() && !"0".equals(card.getCardNumber()) && !card.getSetCode().isEmpty()) { if (!missedCardTFiles.get()) {
CardDownloadData url = new CardDownloadData(card.getName(), card.getSetCode(), card.getCardNumber(), card.usesVariousArt(), if (!card.getCardNumber().isEmpty() && !"0".equals(card.getCardNumber()) && !card.getSetCode().isEmpty()) {
0, "", "", false, card.isDoubleFaced(), card.isNightCard()); CardDownloadData url = new CardDownloadData(card.getName(), card.getSetCode(), card.getCardNumber(), card.usesVariousArt(),
file = new TFile(CardImageUtils.generateImagePath(url)); 0, "", "", false, card.isDoubleFaced(), card.isNightCard());
if (!file.exists()) { TFile file = new TFile(CardImageUtils.generateImagePath(url));
return true; if (!file.exists()) {
missedCardTFiles.set(true);
}
} }
} }
} });
return false; return missedCardTFiles.get();
} }
private void updateCardsToDownload() { private void updateCardsToDownload() {
@ -291,7 +294,7 @@ public class DownloadPictures extends DefaultBoundedRangeModel implements Runnab
/** /**
* read all card names and urls * read all card names and urls
*/ */
ArrayList<CardDownloadData> allCardsUrls = new ArrayList<>(); List<CardDownloadData> allCardsUrls = Collections.synchronizedList(new ArrayList<>());
HashSet<String> ignoreUrls = SettingsManager.getIntance().getIgnoreUrls(); HashSet<String> ignoreUrls = SettingsManager.getIntance().getIgnoreUrls();
/** /**
@ -309,47 +312,47 @@ public class DownloadPictures extends DefaultBoundedRangeModel implements Runnab
try { try {
offlineMode = true; offlineMode = true;
for (CardInfo card : allCards) { allCards.parallelStream().forEach(card -> {
if (!card.getCardNumber().isEmpty() && !"0".equals(card.getCardNumber()) && !card.getSetCode().isEmpty() if (!card.getCardNumber().isEmpty() && !"0".equals(card.getCardNumber()) && !card.getSetCode().isEmpty()
&& !ignoreUrls.contains(card.getSetCode())) { && !ignoreUrls.contains(card.getSetCode())) {
String cardName = card.getName(); String cardName = card.getName();
boolean isType2 = type2SetsFilter.contains(card.getSetCode()); boolean isType2 = type2SetsFilter.contains(card.getSetCode());
CardDownloadData url = new CardDownloadData(cardName, card.getSetCode(), card.getCardNumber(), card.usesVariousArt(), 0, "", "", false, card.isDoubleFaced(), card.isNightCard()); CardDownloadData url = new CardDownloadData(cardName, card.getSetCode(), card.getCardNumber(), card.usesVariousArt(), 0, "", "", false, card.isDoubleFaced(), card.isNightCard());
if (url.getUsesVariousArt()) { if (url.getUsesVariousArt()) {
url.setDownloadName(createDownloadName(card)); url.setDownloadName(createDownloadName(card));
} }
url.setFlipCard(card.isFlipCard()); url.setFlipCard(card.isFlipCard());
url.setSplitCard(card.isSplitCard()); url.setSplitCard(card.isSplitCard());
url.setType2(isType2); url.setType2(isType2);
allCardsUrls.add(url); allCardsUrls.add(url);
if (card.isDoubleFaced()) { if (card.isDoubleFaced()) {
if (card.getSecondSideName() == null || card.getSecondSideName().trim().isEmpty()) { if (card.getSecondSideName() == null || card.getSecondSideName().trim().isEmpty()) {
throw new IllegalStateException("Second side card can't have empty name."); throw new IllegalStateException("Second side card can't have empty name.");
} }
url = new CardDownloadData(card.getSecondSideName(), card.getSetCode(), card.getCardNumber(), card.usesVariousArt(), 0, "", "", false, card.isDoubleFaced(), true); url = new CardDownloadData(card.getSecondSideName(), card.getSetCode(), card.getCardNumber(), card.usesVariousArt(), 0, "", "", false, card.isDoubleFaced(), true);
url.setType2(isType2); url.setType2(isType2);
allCardsUrls.add(url); allCardsUrls.add(url);
} }
if (card.isFlipCard()) { if (card.isFlipCard()) {
if (card.getFlipCardName() == null || card.getFlipCardName().trim().isEmpty()) { if (card.getFlipCardName() == null || card.getFlipCardName().trim().isEmpty()) {
throw new IllegalStateException("Flipped card can't have empty name."); throw new IllegalStateException("Flipped card can't have empty name.");
} }
url = new CardDownloadData(card.getFlipCardName(), card.getSetCode(), card.getCardNumber(), card.usesVariousArt(), 0, "", "", false, card.isDoubleFaced(), card.isNightCard()); url = new CardDownloadData(card.getFlipCardName(), card.getSetCode(), card.getCardNumber(), card.usesVariousArt(), 0, "", "", false, card.isDoubleFaced(), card.isNightCard());
url.setFlipCard(true); url.setFlipCard(true);
url.setFlippedSide(true); url.setFlippedSide(true);
url.setType2(isType2); url.setType2(isType2);
allCardsUrls.add(url); allCardsUrls.add(url);
} }
} else if (card.getCardNumber().isEmpty() || "0".equals(card.getCardNumber())) { } else if (card.getCardNumber().isEmpty() || "0".equals(card.getCardNumber())) {
System.err.println("There was a critical error!"); System.err.println("There was a critical error!");
logger.error("Card has no collector ID and won't be sent to client: " + card); logger.error("Card has no collector ID and won't be sent to client: " + card);
} else if (card.getSetCode().isEmpty()) { } else if (card.getSetCode().isEmpty()) {
System.err.println("There was a critical error!"); System.err.println("There was a critical error!");
logger.error("Card has no set name and won't be sent to client:" + card); logger.error("Card has no set name and won't be sent to client:" + card);
} }
} });
allCardsUrls.addAll(getTokenCardUrls()); allCardsUrls.addAll(getTokenCardUrls());
} catch (Exception e) { } catch (Exception e) {
logger.error(e); logger.error(e);
@ -651,7 +654,7 @@ public class DownloadPictures extends DefaultBoundedRangeModel implements Runnab
boolean useTempFile = false; boolean useTempFile = false;
int responseCode = 0; int responseCode = 0;
URLConnection httpConn = null; URLConnection httpConn = null;
if (temporaryFile != null && temporaryFile.length() > 100) { if (temporaryFile != null && temporaryFile.length() > 100) {
useTempFile = true; useTempFile = true;
} else { } else {
@ -661,7 +664,7 @@ public class DownloadPictures extends DefaultBoundedRangeModel implements Runnab
httpConn.connect(); httpConn.connect();
responseCode = ((HttpURLConnection) httpConn).getResponseCode(); responseCode = ((HttpURLConnection) httpConn).getResponseCode();
} }
if (responseCode == 200 || useTempFile) { if (responseCode == 200 || useTempFile) {
if (!useTempFile) { if (!useTempFile) {
try (BufferedInputStream in = new BufferedInputStream(((HttpURLConnection) httpConn).getInputStream())) { try (BufferedInputStream in = new BufferedInputStream(((HttpURLConnection) httpConn).getInputStream())) {
@ -741,21 +744,21 @@ public class DownloadPictures extends DefaultBoundedRangeModel implements Runnab
httpConn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); httpConn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
httpConn.setRequestProperty("Accept-Encoding", "gzip, deflate, sdch"); httpConn.setRequestProperty("Accept-Encoding", "gzip, deflate, sdch");
httpConn.setRequestProperty("Accept-Language", "en-US,en;q=0.8"); httpConn.setRequestProperty("Accept-Language", "en-US,en;q=0.8");
httpConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36"); httpConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");
break; break;
// ff // ff
case 1: case 1:
httpConn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); httpConn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
httpConn.setRequestProperty("Accept-Encoding", "gzip, deflate"); httpConn.setRequestProperty("Accept-Encoding", "gzip, deflate");
httpConn.setRequestProperty("Accept-Language", "en-US;q=0.5,en;q=0.3"); httpConn.setRequestProperty("Accept-Language", "en-US;q=0.5,en;q=0.3");
httpConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:44.0) Gecko/20100101 Firefox/44.0"); httpConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:50.0) Gecko/20100101 Firefox/50.0");
break; break;
// ie // ie
case 2: case 2:
httpConn.setRequestProperty("Accept", "text/html, application/xhtml+xml, */*"); httpConn.setRequestProperty("Accept", "text/html, application/xhtml+xml, */*");
httpConn.setRequestProperty("Accept-Encoding", "gzip, deflate"); httpConn.setRequestProperty("Accept-Encoding", "gzip, deflate");
httpConn.setRequestProperty("Accept-Language", "en-US"); httpConn.setRequestProperty("Accept-Language", "en-US");
httpConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko"); httpConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko");
break; break;
} }
} }
@ -789,14 +792,16 @@ public class DownloadPictures extends DefaultBoundedRangeModel implements Runnab
bar.setString(String.format("%d of %d cards finished! Please wait! [%.1f Mb]", bar.setString(String.format("%d of %d cards finished! Please wait! [%.1f Mb]",
card, count, mb)); card, count, mb));
} else { } else {
Iterator<CardDownloadData> cardsIterator = DownloadPictures.this.cards.iterator(); List<CardDownloadData> remainingCards = Collections.synchronizedList(new ArrayList<>());
while (cardsIterator.hasNext()) { DownloadPictures.this.cards.parallelStream().forEach(cardDownloadData -> {
CardDownloadData cardDownloadData = cardsIterator.next();
TFile file = new TFile(CardImageUtils.generateImagePath(cardDownloadData)); TFile file = new TFile(CardImageUtils.generateImagePath(cardDownloadData));
if (file.exists()) { if (!file.exists()) {
cardsIterator.remove(); remainingCards.add(cardDownloadData);
} }
} });
DownloadPictures.this.cards = new ArrayList<>(remainingCards);
count = DownloadPictures.this.cards.size(); count = DownloadPictures.this.cards.size();
if (count == 0) { if (count == 0) {