From adb666587bd8bb0d00f053e567f233f8cef6b498 Mon Sep 17 00:00:00 2001 From: Oleg Agafonov Date: Mon, 27 May 2019 10:19:47 +0400 Subject: [PATCH] Refactor images download for #5814 --- .../plugins/card/dl/DownloadServiceInfo.java | 19 +++ .../sources/AltMtgOnlTokensImageSource.java | 7 + .../card/dl/sources/CardImageSource.java | 4 + .../card/dl/sources/CopyPasteImageSource.java | 12 +- .../card/dl/sources/GrabbagImageSource.java | 11 +- .../card/dl/sources/MagidexImageSource.java | 6 + .../card/dl/sources/MtgImageSource.java | 7 + .../dl/sources/MtgOnlTokensImageSource.java | 7 + .../dl/sources/MythicspoilerComSource.java | 6 + .../card/dl/sources/ScryfallImageSource.java | 41 ++++- .../card/dl/sources/TokensMtgImageSource.java | 13 +- .../dl/sources/WizardCardsImageSource.java | 6 + .../card/images/DownloadPicturesService.java | 148 ++++++++++-------- 13 files changed, 204 insertions(+), 83 deletions(-) create mode 100644 Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadServiceInfo.java diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadServiceInfo.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadServiceInfo.java new file mode 100644 index 0000000000..d125845bce --- /dev/null +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadServiceInfo.java @@ -0,0 +1,19 @@ +package org.mage.plugins.card.dl; + +import java.net.Proxy; + +/** + * @author JayDi85 + */ +public interface DownloadServiceInfo { + + Proxy getProxy(); + + boolean isNeedCancel(); + + void incErrorCount(); + + void updateMessage(String text); + + void showDownloadControls(boolean needToShow); +} diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/AltMtgOnlTokensImageSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/AltMtgOnlTokensImageSource.java index f8216028ba..2819de9b4e 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/AltMtgOnlTokensImageSource.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/AltMtgOnlTokensImageSource.java @@ -1,10 +1,12 @@ package org.mage.plugins.card.dl.sources; import org.apache.log4j.Logger; +import org.mage.plugins.card.dl.DownloadServiceInfo; import org.mage.plugins.card.images.CardDownloadData; import java.io.IOException; import java.util.HashMap; +import java.util.List; /** * @author spjspj @@ -57,6 +59,11 @@ public enum AltMtgOnlTokensImageSource implements CardImageSource { return null; } + @Override + public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List downloadList) { + return true; + } + @Override public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception { return null; diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CardImageSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CardImageSource.java index 758f38881e..1106da877e 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CardImageSource.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CardImageSource.java @@ -1,9 +1,11 @@ package org.mage.plugins.card.dl.sources; import mage.client.util.CardLanguage; +import org.mage.plugins.card.dl.DownloadServiceInfo; import org.mage.plugins.card.images.CardDownloadData; import java.util.ArrayList; +import java.util.List; /** * @author North, JayDi85 @@ -14,6 +16,8 @@ public interface CardImageSource { CardImageUrls generateTokenUrl(CardDownloadData card) throws Exception; + boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List downloadList); + String getNextHttpImageUrl(); String getFileForHttpImage(String httpImageUrl); diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CopyPasteImageSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CopyPasteImageSource.java index 766208bdf1..f930bafeed 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CopyPasteImageSource.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CopyPasteImageSource.java @@ -1,6 +1,7 @@ package org.mage.plugins.card.dl.sources; import mage.cards.Sets; +import org.mage.plugins.card.dl.DownloadServiceInfo; import org.mage.plugins.card.images.CardDownloadData; import javax.swing.*; @@ -8,10 +9,8 @@ import java.awt.*; import java.awt.datatransfer.Clipboard; import java.awt.datatransfer.StringSelection; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.Set; +import java.util.List; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -73,6 +72,11 @@ public enum CopyPasteImageSource implements CardImageSource { return null; } + @Override + public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List downloadList) { + return true; + } + @Override public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception { if (singleLinks == null) { diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/GrabbagImageSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/GrabbagImageSource.java index b8c9e2b65b..0bc422ef18 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/GrabbagImageSource.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/GrabbagImageSource.java @@ -1,13 +1,11 @@ package org.mage.plugins.card.dl.sources; import org.apache.log4j.Logger; +import org.mage.plugins.card.dl.DownloadServiceInfo; import org.mage.plugins.card.images.CardDownloadData; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.Set; +import java.util.*; import java.util.concurrent.TimeUnit; import java.util.logging.Level; @@ -46,6 +44,11 @@ public enum GrabbagImageSource implements CardImageSource { return null; } + @Override + public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List downloadList) { + return true; + } + @Override public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception { if (singleLinks == null) { diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MagidexImageSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MagidexImageSource.java index 4bd2c49433..098ceecf02 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MagidexImageSource.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MagidexImageSource.java @@ -1,5 +1,6 @@ package org.mage.plugins.card.dl.sources; +import org.mage.plugins.card.dl.DownloadServiceInfo; import org.mage.plugins.card.images.CardDownloadData; import java.net.URI; @@ -227,6 +228,11 @@ public enum MagidexImageSource implements CardImageSource { return null; } + @Override + public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List downloadList) { + return true; + } + @Override public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception { String cardDownloadName = card.getDownloadName().toLowerCase(Locale.ENGLISH); diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MtgImageSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MtgImageSource.java index 7c8a368579..8509e95730 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MtgImageSource.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MtgImageSource.java @@ -1,7 +1,9 @@ package org.mage.plugins.card.dl.sources; +import org.mage.plugins.card.dl.DownloadServiceInfo; import org.mage.plugins.card.images.CardDownloadData; +import java.util.List; import java.util.Locale; /** @@ -28,6 +30,11 @@ public enum MtgImageSource implements CardImageSource { return null; } + @Override + public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List downloadList) { + return true; + } + @Override public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception { String collectorId = card.getCollectorId(); diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MtgOnlTokensImageSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MtgOnlTokensImageSource.java index f43baec92c..ed7f174664 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MtgOnlTokensImageSource.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MtgOnlTokensImageSource.java @@ -1,10 +1,12 @@ package org.mage.plugins.card.dl.sources; import org.apache.log4j.Logger; +import org.mage.plugins.card.dl.DownloadServiceInfo; import org.mage.plugins.card.images.CardDownloadData; import java.io.IOException; import java.util.HashMap; +import java.util.List; /** * @author spjspj @@ -57,6 +59,11 @@ public enum MtgOnlTokensImageSource implements CardImageSource { return null; } + @Override + public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List downloadList) { + return true; + } + @Override public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception { return null; diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MythicspoilerComSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MythicspoilerComSource.java index 1c0ecaf857..c8b30400d3 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MythicspoilerComSource.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MythicspoilerComSource.java @@ -7,6 +7,7 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.mage.plugins.card.dl.DownloadServiceInfo; import org.mage.plugins.card.images.CardDownloadData; import java.io.BufferedReader; @@ -381,6 +382,11 @@ public enum MythicspoilerComSource implements CardImageSource { return pageLinks; } + @Override + public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List downloadList) { + return true; + } + @Override public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception { String collectorId = card.getCollectorId(); diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java index 133bebf5b0..82dd5aeecf 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java @@ -1,12 +1,12 @@ package org.mage.plugins.card.dl.sources; import mage.client.util.CardLanguage; +import org.apache.log4j.Logger; +import org.mage.plugins.card.dl.DownloadServiceInfo; import org.mage.plugins.card.images.CardDownloadData; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; +import java.net.Proxy; +import java.util.*; /** * @author JayDi85 @@ -15,8 +15,11 @@ public enum ScryfallImageSource implements CardImageSource { instance; + private static final Logger logger = Logger.getLogger(ScryfallImageSource.class); + private final Map languageAliases; private CardLanguage currentLanguage = CardLanguage.ENGLISH; // working language + private Map preparedUrls = new HashMap<>(); ScryfallImageSource() { // LANGUAGES @@ -36,6 +39,11 @@ public enum ScryfallImageSource implements CardImageSource { } private CardImageUrls innerGenerateURL(CardDownloadData card, boolean isToken) { + String prepared = preparedUrls.getOrDefault(card, null); + if (prepared != null) { + return new CardImageUrls(prepared, null); + } + String defaultCode = CardLanguage.ENGLISH.getCode(); String localizedCode = languageAliases.getOrDefault(this.getCurrentLanguage(), defaultCode); // loc example: https://api.scryfall.com/cards/xln/121/ru?format=image @@ -99,6 +107,31 @@ public enum ScryfallImageSource implements CardImageSource { return new CardImageUrls(baseUrl, alternativeUrl); } + @Override + public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List downloadList) { + // prepare download list example ( + Proxy proxy = downloadServiceInfo.getProxy(); + + preparedUrls.clear(); + for (CardDownloadData card : downloadList) { + // need cancel + if (downloadServiceInfo.isNeedCancel()) { + return false; + } + + // TODO: download faces info here + if (card.isTwoFacedCard()) { + String url = null; + preparedUrls.put(card, url); + } + + // inc error count to stop on too many errors + // downloadServiceInfo.incErrorCount(); + } + + return true; + } + @Override public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception { return innerGenerateURL(card, false); diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/TokensMtgImageSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/TokensMtgImageSource.java index 5ac33a0b0a..ad8f654f47 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/TokensMtgImageSource.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/TokensMtgImageSource.java @@ -2,6 +2,7 @@ package org.mage.plugins.card.dl.sources; import mage.constants.SubType; import org.apache.log4j.Logger; +import org.mage.plugins.card.dl.DownloadServiceInfo; import org.mage.plugins.card.images.CardDownloadData; import org.mage.plugins.card.images.DownloadPicturesService; import org.mage.plugins.card.utils.CardImageUtils; @@ -50,6 +51,11 @@ public enum TokensMtgImageSource implements CardImageSource { return null; } + @Override + public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List downloadList) { + return true; + } + @Override public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception { return null; @@ -180,7 +186,7 @@ public enum TokensMtgImageSource implements CardImageSource { private HashMap> getTokensData() throws IOException { synchronized (tokensDataSync) { if (tokensData == null) { - DownloadPicturesService.getInstance().updateAndViewMessage("Find tokens data..."); + DownloadPicturesService.getInstance().updateMessage("Find tokens data..."); tokensData = new HashMap<>(); // get tokens data from resource file @@ -231,10 +237,11 @@ public enum TokensMtgImageSource implements CardImageSource { } } } - DownloadPicturesService.getInstance().updateAndViewMessage(""); + DownloadPicturesService.getInstance().updateMessage(""); + DownloadPicturesService.getInstance().showDownloadControls(true); } catch (Exception ex) { logger.warn("Failed to get tokens description from tokens.mtg.onl", ex); - DownloadPicturesService.getInstance().updateAndViewMessage(ex.getMessage()); + DownloadPicturesService.getInstance().updateMessage(ex.getMessage()); } } } diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/WizardCardsImageSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/WizardCardsImageSource.java index 0d434ec1c0..3ee788d1e5 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/WizardCardsImageSource.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/WizardCardsImageSource.java @@ -9,6 +9,7 @@ import org.apache.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.mage.plugins.card.dl.DownloadServiceInfo; import org.mage.plugins.card.images.CardDownloadData; import org.mage.plugins.card.utils.CardImageUtils; @@ -451,6 +452,11 @@ public enum WizardCardsImageSource implements CardImageSource { return null; } + @Override + public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List downloadList) { + return true; + } + @Override public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception { String collectorId = card.getCollectorId(); diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/images/DownloadPicturesService.java b/Mage.Client/src/main/java/org/mage/plugins/card/images/DownloadPicturesService.java index 2ffc11bad1..13dd0a9883 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/images/DownloadPicturesService.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/images/DownloadPicturesService.java @@ -16,6 +16,7 @@ import net.java.truevfs.access.TFileOutputStream; import net.java.truevfs.access.TVFS; import net.java.truevfs.kernel.spec.FsSyncException; import org.apache.log4j.Logger; +import org.mage.plugins.card.dl.DownloadServiceInfo; import org.mage.plugins.card.dl.sources.*; import org.mage.plugins.card.utils.CardImageUtils; @@ -37,7 +38,7 @@ import static org.mage.plugins.card.utils.CardImageUtils.getImagesDir; /** * @author JayDi85 */ -public class DownloadPicturesService extends DefaultBoundedRangeModel implements Runnable { +public class DownloadPicturesService extends DefaultBoundedRangeModel implements DownloadServiceInfo, Runnable { // don't forget to remove new sets from ignore.urls to download (properties file in resources) private static DownloadPicturesService instance; @@ -65,7 +66,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements private static CardImageSource selectedSource; private final Object sync = new Object(); - private Proxy p = Proxy.NO_PROXY; + private Proxy proxy = Proxy.NO_PROXY; enum DownloadSources { WIZARDS("1. wizards.com - low quality CARDS, multi-language, slow download", WizardCardsImageSource.instance), @@ -118,7 +119,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements instance.setNeedCancel(true); } - private boolean getNeedCancel() { + public boolean isNeedCancel() { return this.needCancel || (this.errorCount > MAX_ERRORS_COUNT_BEFORE_CANCEL); } @@ -126,7 +127,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements this.needCancel = needCancel; } - private void incErrorCount() { + public void incErrorCount() { this.errorCount = this.errorCount + 1; } @@ -196,23 +197,24 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements } public void findMissingCards() { - updateAndViewMessage("Loading..."); + updateMessage("Loading..."); this.cardsAll.clear(); this.cardsMissing.clear(); this.cardsDownloadQueue.clear(); - updateAndViewMessage("Loading cards list..."); + updateMessage("Loading cards list..."); this.cardsAll = Collections.synchronizedList(CardRepository.instance.findCards(new CardCriteria())); - updateAndViewMessage("Finding missing images..."); + updateMessage("Finding missing images..."); this.cardsMissing = prepareMissingCards(this.cardsAll, uiDialog.getRedownloadCheckbox().isSelected()); - updateAndViewMessage("Finding available sets from selected source..."); + updateMessage("Finding available sets from selected source..."); this.uiDialog.getSetsCombo().setModel(new DefaultComboBoxModel<>(getSetsForCurrentImageSource())); reloadCardsToDownload(this.uiDialog.getSetsCombo().getSelectedItem().toString()); this.uiDialog.showDownloadControls(true); - updateAndViewMessage(""); + updateMessage(""); + showDownloadControls(true); } private void reloadLanguagesForSelectedSource() { @@ -237,13 +239,13 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements } } - public void updateAndViewMessage(String text) { + public void updateMessage(String text) { this.uiDialog.setGlobalInfo(text); + } - // auto-size on empty message (on complete) - if (text.isEmpty()) { - this.uiDialog.showDownloadControls(true); - } + public void showDownloadControls(boolean needToShow) { + // auto-size form on show + this.uiDialog.showDownloadControls(needToShow); } private String getSetNameWithYear(ExpansionSet exp) { @@ -587,7 +589,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements break; case NONE: default: - p = Proxy.NO_PROXY; + proxy = Proxy.NO_PROXY; break; } @@ -595,67 +597,69 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements try { String address = PreferencesDialog.getCachedValue(PreferencesDialog.KEY_PROXY_ADDRESS, ""); Integer port = Integer.parseInt(PreferencesDialog.getCachedValue(PreferencesDialog.KEY_PROXY_PORT, "80")); - p = new Proxy(type, new InetSocketAddress(address, port)); + proxy = new Proxy(type, new InetSocketAddress(address, port)); } catch (Exception ex) { throw new RuntimeException("Gui_DownloadPicturesService : error 1 - " + ex); } } - if (p != null) { - update(0, cardsDownloadQueue.size()); + if (proxy != null) { logger.info("Started download of " + cardsDownloadQueue.size() + " images" + " from source: " + selectedSource.getSourceName() + ", language: " + selectedSource.getCurrentLanguage().getCode()); + uiDialog.getProgressBar().setString("Preparing download list..."); + if (selectedSource.prepareDownloadList(this, cardsDownloadQueue)) { + update(0, cardsDownloadQueue.size()); + int numberOfThreads = Integer.parseInt(PreferencesDialog.getCachedValue(PreferencesDialog.KEY_CARD_IMAGES_THREADS, "10")); + ExecutorService executor = Executors.newFixedThreadPool(numberOfThreads); + for (int i = 0; i < cardsDownloadQueue.size() && !this.isNeedCancel(); i++) { + try { + CardDownloadData card = cardsDownloadQueue.get(i); - int numberOfThreads = Integer.parseInt(PreferencesDialog.getCachedValue(PreferencesDialog.KEY_CARD_IMAGES_THREADS, "10")); - ExecutorService executor = Executors.newFixedThreadPool(numberOfThreads); - for (int i = 0; i < cardsDownloadQueue.size() && !this.getNeedCancel(); i++) { - try { - CardDownloadData card = cardsDownloadQueue.get(i); + logger.debug("Downloading image: " + card.getName() + " (" + card.getSet() + ')'); - logger.debug("Downloading image: " + card.getName() + " (" + card.getSet() + ')'); - - CardImageUrls urls; - if (card.isToken()) { - if (!"0".equals(card.getCollectorId())) { - continue; - } - urls = selectedSource.generateTokenUrl(card); - } else { - urls = selectedSource.generateCardUrl(card); - } - - if (urls == null) { - String imageRef = selectedSource.getNextHttpImageUrl(); - String fileName = selectedSource.getFileForHttpImage(imageRef); - if (imageRef != null && fileName != null) { - imageRef = selectedSource.getSourceName() + imageRef; - try { - card.setToken(selectedSource.isTokenSource()); - Runnable task = new DownloadTask(card, imageRef, fileName, selectedSource.getTotalImages()); - executor.execute(task); - } catch (Exception ex) { - } - } else if (selectedSource.getTotalImages() == -1) { - logger.info("Image not available on " + selectedSource.getSourceName() + ": " + card.getName() + " (" + card.getSet() + ')'); - synchronized (sync) { - update(cardIndex + 1, cardsDownloadQueue.size()); + CardImageUrls urls; + if (card.isToken()) { + if (!"0".equals(card.getCollectorId())) { + continue; } + urls = selectedSource.generateTokenUrl(card); + } else { + urls = selectedSource.generateCardUrl(card); } - } else { - Runnable task = new DownloadTask(card, urls, cardsDownloadQueue.size()); - executor.execute(task); + + if (urls == null) { + String imageRef = selectedSource.getNextHttpImageUrl(); + String fileName = selectedSource.getFileForHttpImage(imageRef); + if (imageRef != null && fileName != null) { + imageRef = selectedSource.getSourceName() + imageRef; + try { + card.setToken(selectedSource.isTokenSource()); + Runnable task = new DownloadTask(card, imageRef, fileName, selectedSource.getTotalImages()); + executor.execute(task); + } catch (Exception ex) { + } + } else if (selectedSource.getTotalImages() == -1) { + logger.info("Image not available on " + selectedSource.getSourceName() + ": " + card.getName() + " (" + card.getSet() + ')'); + synchronized (sync) { + update(cardIndex + 1, cardsDownloadQueue.size()); + } + } + } else { + Runnable task = new DownloadTask(card, urls, cardsDownloadQueue.size()); + executor.execute(task); + } + } catch (Exception ex) { + logger.error(ex, ex); } - } catch (Exception ex) { - logger.error(ex, ex); } - } - executor.shutdown(); - while (!executor.isTerminated()) { - try { - TimeUnit.SECONDS.sleep(1); - } catch (InterruptedException ie) { + executor.shutdown(); + while (!executor.isTerminated()) { + try { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException ie) { + } } } } @@ -671,6 +675,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements // stop reloadCardsToDownload(uiDialog.getSetsCombo().getSelectedItem().toString()); + enableDialogButtons(); // reset images cache ImageCache.clearCache(); @@ -707,7 +712,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements @Override public void run() { - if (DownloadPicturesService.getInstance().getNeedCancel()) { + if (DownloadPicturesService.getInstance().isNeedCancel()) { synchronized (sync) { update(cardIndex + 1, count); } @@ -791,14 +796,14 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements URL url = new URL(currentUrl); // on download cancel need to stop - if (DownloadPicturesService.getInstance().getNeedCancel()) { + if (DownloadPicturesService.getInstance().isNeedCancel()) { return; } // download selectedSource.doPause(url.getPath()); - httpConn = url.openConnection(p); + httpConn = url.openConnection(proxy); if (httpConn != null) { httpConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9.2.2) Gecko/20100316 Firefox/3.6.2"); @@ -847,7 +852,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements int len; while ((len = in.read(buf)) != -1) { // user cancelled - if (DownloadPicturesService.getInstance().getNeedCancel()) { + if (DownloadPicturesService.getInstance().isNeedCancel()) { // stop download, save current state and exit TFile archive = destFile.getTopLevelArchive(); ///* not need to unmout/close - it's auto action @@ -934,14 +939,21 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements // try download again } - this.uiDialog.getRedownloadCheckbox().setSelected(false); - uiDialog.enableActionControls(true); - uiDialog.getStartButton().setEnabled(true); + enableDialogButtons(); } } private static final long serialVersionUID = 1L; + private void enableDialogButtons() { + uiDialog.getRedownloadCheckbox().setSelected(false); // reset re-download button after finished + uiDialog.enableActionControls(true); + uiDialog.getStartButton().setEnabled(true); + } + + public Proxy getProxy() { + return proxy; + } } class LoadMissingCardDataNew implements Runnable {