github/mage
1
0
Fork 0
mirror of https://github.com/correl/mage.git synced 2024-12-24 11:50:45 +00:00
Code Issues Projects Releases Packages Wiki Activity

Refactor images download for #5814

Browse source
This commit is contained in:
Oleg Agafonov 2019-05-27 10:19:47 +04:00
parent bf528dfc59
commit adb666587b
13 changed files with 204 additions and 83 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadServiceInfo.java Normal file
View file

@ -0,0 +1,19 @@
package org.mage.plugins.card.dl;
import java.net.Proxy;
/**
* @author JayDi85
*/
public interface DownloadServiceInfo {
Proxy getProxy();
boolean isNeedCancel();
void incErrorCount();
void updateMessage(String text);
void showDownloadControls(boolean needToShow);
}

7
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/AltMtgOnlTokensImageSource.java
View file

@ -1,10 +1,12 @@
package org.mage.plugins.card.dl.sources;
import org.apache.log4j.Logger;
import org.mage.plugins.card.dl.DownloadServiceInfo;
import org.mage.plugins.card.images.CardDownloadData;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
/**
* @author spjspj
@ -57,6 +59,11 @@ public enum AltMtgOnlTokensImageSource implements CardImageSource {
return null;
}
@Override
public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) {
return true;
}
@Override
public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception {
return null;

4
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CardImageSource.java
View file

@ -1,9 +1,11 @@
package org.mage.plugins.card.dl.sources;
import mage.client.util.CardLanguage;
import org.mage.plugins.card.dl.DownloadServiceInfo;
import org.mage.plugins.card.images.CardDownloadData;
import java.util.ArrayList;
import java.util.List;
/**
* @author North, JayDi85
@ -14,6 +16,8 @@ public interface CardImageSource {
CardImageUrls generateTokenUrl(CardDownloadData card) throws Exception;
boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList);
String getNextHttpImageUrl();
String getFileForHttpImage(String httpImageUrl);

12
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CopyPasteImageSource.java
View file

@ -1,6 +1,7 @@
package org.mage.plugins.card.dl.sources;
import mage.cards.Sets;
import org.mage.plugins.card.dl.DownloadServiceInfo;
import org.mage.plugins.card.images.CardDownloadData;
import javax.swing.*;
@ -8,10 +9,8 @@ import java.awt.*;
import java.awt.datatransfer.Clipboard;
import java.awt.datatransfer.StringSelection;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.Set;
import java.util.List;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -73,6 +72,11 @@ public enum CopyPasteImageSource implements CardImageSource {
return null;
}
@Override
public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) {
return true;
}
@Override
public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception {
if (singleLinks == null) {

11
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/GrabbagImageSource.java
View file

@ -1,13 +1,11 @@
package org.mage.plugins.card.dl.sources;
import org.apache.log4j.Logger;
import org.mage.plugins.card.dl.DownloadServiceInfo;
import org.mage.plugins.card.images.CardDownloadData;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.Set;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
@ -46,6 +44,11 @@ public enum GrabbagImageSource implements CardImageSource {
return null;
}
@Override
public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) {
return true;
}
@Override
public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception {
if (singleLinks == null) {

6
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MagidexImageSource.java
View file

@ -1,5 +1,6 @@
package org.mage.plugins.card.dl.sources;
import org.mage.plugins.card.dl.DownloadServiceInfo;
import org.mage.plugins.card.images.CardDownloadData;
import java.net.URI;
@ -227,6 +228,11 @@ public enum MagidexImageSource implements CardImageSource {
return null;
}
@Override
public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) {
return true;
}
@Override
public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception {
String cardDownloadName = card.getDownloadName().toLowerCase(Locale.ENGLISH);

7
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MtgImageSource.java
View file

@ -1,7 +1,9 @@
package org.mage.plugins.card.dl.sources;
import org.mage.plugins.card.dl.DownloadServiceInfo;
import org.mage.plugins.card.images.CardDownloadData;
import java.util.List;
import java.util.Locale;
/**
@ -28,6 +30,11 @@ public enum MtgImageSource implements CardImageSource {
return null;
}
@Override
public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) {
return true;
}
@Override
public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception {
String collectorId = card.getCollectorId();

7
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MtgOnlTokensImageSource.java
View file

@ -1,10 +1,12 @@
package org.mage.plugins.card.dl.sources;
import org.apache.log4j.Logger;
import org.mage.plugins.card.dl.DownloadServiceInfo;
import org.mage.plugins.card.images.CardDownloadData;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
/**
* @author spjspj
@ -57,6 +59,11 @@ public enum MtgOnlTokensImageSource implements CardImageSource {
return null;
}
@Override
public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) {
return true;
}
@Override
public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception {
return null;

6
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/MythicspoilerComSource.java
View file

@ -7,6 +7,7 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.mage.plugins.card.dl.DownloadServiceInfo;
import org.mage.plugins.card.images.CardDownloadData;
import java.io.BufferedReader;
@ -381,6 +382,11 @@ public enum MythicspoilerComSource implements CardImageSource {
return pageLinks;
}
@Override
public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) {
return true;
}
@Override
public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception {
String collectorId = card.getCollectorId();

41
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java
View file

@ -1,12 +1,12 @@
package org.mage.plugins.card.dl.sources;
import mage.client.util.CardLanguage;
import org.apache.log4j.Logger;
import org.mage.plugins.card.dl.DownloadServiceInfo;
import org.mage.plugins.card.images.CardDownloadData;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.net.Proxy;
import java.util.*;
/**
* @author JayDi85
@ -15,8 +15,11 @@ public enum ScryfallImageSource implements CardImageSource {
instance;
private static final Logger logger = Logger.getLogger(ScryfallImageSource.class);
private final Map<CardLanguage, String> languageAliases;
private CardLanguage currentLanguage = CardLanguage.ENGLISH; // working language
private Map<CardDownloadData, String> preparedUrls = new HashMap<>();
ScryfallImageSource() {
// LANGUAGES
@ -36,6 +39,11 @@ public enum ScryfallImageSource implements CardImageSource {
}
private CardImageUrls innerGenerateURL(CardDownloadData card, boolean isToken) {
String prepared = preparedUrls.getOrDefault(card, null);
if (prepared != null) {
return new CardImageUrls(prepared, null);
}
String defaultCode = CardLanguage.ENGLISH.getCode();
String localizedCode = languageAliases.getOrDefault(this.getCurrentLanguage(), defaultCode);
// loc example: https://api.scryfall.com/cards/xln/121/ru?format=image
@ -99,6 +107,31 @@ public enum ScryfallImageSource implements CardImageSource {
return new CardImageUrls(baseUrl, alternativeUrl);
}
@Override
public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) {
// prepare download list example (
Proxy proxy = downloadServiceInfo.getProxy();
preparedUrls.clear();
for (CardDownloadData card : downloadList) {
// need cancel
if (downloadServiceInfo.isNeedCancel()) {
return false;
}
// TODO: download faces info here
if (card.isTwoFacedCard()) {
String url = null;
preparedUrls.put(card, url);
}
// inc error count to stop on too many errors
// downloadServiceInfo.incErrorCount();
}
return true;
}
@Override
public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception {
return innerGenerateURL(card, false);

13
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/TokensMtgImageSource.java
View file

@ -2,6 +2,7 @@ package org.mage.plugins.card.dl.sources;
import mage.constants.SubType;
import org.apache.log4j.Logger;
import org.mage.plugins.card.dl.DownloadServiceInfo;
import org.mage.plugins.card.images.CardDownloadData;
import org.mage.plugins.card.images.DownloadPicturesService;
import org.mage.plugins.card.utils.CardImageUtils;
@ -50,6 +51,11 @@ public enum TokensMtgImageSource implements CardImageSource {
return null;
}
@Override
public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) {
return true;
}
@Override
public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception {
return null;
@ -180,7 +186,7 @@ public enum TokensMtgImageSource implements CardImageSource {
private HashMap<String, List<TokenData>> getTokensData() throws IOException {
synchronized (tokensDataSync) {
if (tokensData == null) {
DownloadPicturesService.getInstance().updateAndViewMessage("Find tokens data...");
DownloadPicturesService.getInstance().updateMessage("Find tokens data...");
tokensData = new HashMap<>();
// get tokens data from resource file
@ -231,10 +237,11 @@ public enum TokensMtgImageSource implements CardImageSource {
}
}
}
DownloadPicturesService.getInstance().updateAndViewMessage("");
DownloadPicturesService.getInstance().updateMessage("");
DownloadPicturesService.getInstance().showDownloadControls(true);
} catch (Exception ex) {
logger.warn("Failed to get tokens description from tokens.mtg.onl", ex);
DownloadPicturesService.getInstance().updateAndViewMessage(ex.getMessage());
DownloadPicturesService.getInstance().updateMessage(ex.getMessage());
}
}
}

6
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/WizardCardsImageSource.java
View file

@ -9,6 +9,7 @@ import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.mage.plugins.card.dl.DownloadServiceInfo;
import org.mage.plugins.card.images.CardDownloadData;
import org.mage.plugins.card.utils.CardImageUtils;
@ -451,6 +452,11 @@ public enum WizardCardsImageSource implements CardImageSource {
return null;
}
@Override
public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) {
return true;
}
@Override
public CardImageUrls generateCardUrl(CardDownloadData card) throws Exception {
String collectorId = card.getCollectorId();

148
Mage.Client/src/main/java/org/mage/plugins/card/images/DownloadPicturesService.java
View file

@ -16,6 +16,7 @@ import net.java.truevfs.access.TFileOutputStream;
import net.java.truevfs.access.TVFS;
import net.java.truevfs.kernel.spec.FsSyncException;
import org.apache.log4j.Logger;
import org.mage.plugins.card.dl.DownloadServiceInfo;
import org.mage.plugins.card.dl.sources.*;
import org.mage.plugins.card.utils.CardImageUtils;
@ -37,7 +38,7 @@ import static org.mage.plugins.card.utils.CardImageUtils.getImagesDir;
/**
* @author JayDi85
*/
public class DownloadPicturesService extends DefaultBoundedRangeModel implements Runnable {
public class DownloadPicturesService extends DefaultBoundedRangeModel implements DownloadServiceInfo, Runnable {
// don't forget to remove new sets from ignore.urls to download (properties file in resources)
private static DownloadPicturesService instance;
@ -65,7 +66,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
private static CardImageSource selectedSource;
private final Object sync = new Object();
private Proxy p = Proxy.NO_PROXY;
private Proxy proxy = Proxy.NO_PROXY;
enum DownloadSources {
WIZARDS("1. wizards.com - low quality CARDS, multi-language, slow download", WizardCardsImageSource.instance),
@ -118,7 +119,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
instance.setNeedCancel(true);
}
private boolean getNeedCancel() {
public boolean isNeedCancel() {
return this.needCancel || (this.errorCount > MAX_ERRORS_COUNT_BEFORE_CANCEL);
}
@ -126,7 +127,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
this.needCancel = needCancel;
}
private void incErrorCount() {
public void incErrorCount() {
this.errorCount = this.errorCount + 1;
}
@ -196,23 +197,24 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
}
public void findMissingCards() {
updateAndViewMessage("Loading...");
updateMessage("Loading...");
this.cardsAll.clear();
this.cardsMissing.clear();
this.cardsDownloadQueue.clear();
updateAndViewMessage("Loading cards list...");
updateMessage("Loading cards list...");
this.cardsAll = Collections.synchronizedList(CardRepository.instance.findCards(new CardCriteria()));
updateAndViewMessage("Finding missing images...");
updateMessage("Finding missing images...");
this.cardsMissing = prepareMissingCards(this.cardsAll, uiDialog.getRedownloadCheckbox().isSelected());
updateAndViewMessage("Finding available sets from selected source...");
updateMessage("Finding available sets from selected source...");
this.uiDialog.getSetsCombo().setModel(new DefaultComboBoxModel<>(getSetsForCurrentImageSource()));
reloadCardsToDownload(this.uiDialog.getSetsCombo().getSelectedItem().toString());
this.uiDialog.showDownloadControls(true);
updateAndViewMessage("");
updateMessage("");
showDownloadControls(true);
}
private void reloadLanguagesForSelectedSource() {
@ -237,13 +239,13 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
}
}
public void updateAndViewMessage(String text) {
public void updateMessage(String text) {
this.uiDialog.setGlobalInfo(text);
}
// auto-size on empty message (on complete)
if (text.isEmpty()) {
this.uiDialog.showDownloadControls(true);
}
public void showDownloadControls(boolean needToShow) {
// auto-size form on show
this.uiDialog.showDownloadControls(needToShow);
}
private String getSetNameWithYear(ExpansionSet exp) {
@ -587,7 +589,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
break;
case NONE:
default:
p = Proxy.NO_PROXY;
proxy = Proxy.NO_PROXY;
break;
}
@ -595,67 +597,69 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
try {
String address = PreferencesDialog.getCachedValue(PreferencesDialog.KEY_PROXY_ADDRESS, "");
Integer port = Integer.parseInt(PreferencesDialog.getCachedValue(PreferencesDialog.KEY_PROXY_PORT, "80"));
p = new Proxy(type, new InetSocketAddress(address, port));
proxy = new Proxy(type, new InetSocketAddress(address, port));
} catch (Exception ex) {
throw new RuntimeException("Gui_DownloadPicturesService : error 1 - " + ex);
}
}
if (p != null) {
update(0, cardsDownloadQueue.size());
if (proxy != null) {
logger.info("Started download of " + cardsDownloadQueue.size() + " images"
+ " from source: " + selectedSource.getSourceName()
+ ", language: " + selectedSource.getCurrentLanguage().getCode());
uiDialog.getProgressBar().setString("Preparing download list...");
if (selectedSource.prepareDownloadList(this, cardsDownloadQueue)) {
update(0, cardsDownloadQueue.size());
int numberOfThreads = Integer.parseInt(PreferencesDialog.getCachedValue(PreferencesDialog.KEY_CARD_IMAGES_THREADS, "10"));
ExecutorService executor = Executors.newFixedThreadPool(numberOfThreads);
for (int i = 0; i < cardsDownloadQueue.size() && !this.isNeedCancel(); i++) {
try {
CardDownloadData card = cardsDownloadQueue.get(i);
int numberOfThreads = Integer.parseInt(PreferencesDialog.getCachedValue(PreferencesDialog.KEY_CARD_IMAGES_THREADS, "10"));
ExecutorService executor = Executors.newFixedThreadPool(numberOfThreads);
for (int i = 0; i < cardsDownloadQueue.size() && !this.getNeedCancel(); i++) {
try {
CardDownloadData card = cardsDownloadQueue.get(i);
logger.debug("Downloading image: " + card.getName() + " (" + card.getSet() + ')');
logger.debug("Downloading image: " + card.getName() + " (" + card.getSet() + ')');
CardImageUrls urls;
if (card.isToken()) {
if (!"0".equals(card.getCollectorId())) {
continue;
}
urls = selectedSource.generateTokenUrl(card);
} else {
urls = selectedSource.generateCardUrl(card);
}
if (urls == null) {
String imageRef = selectedSource.getNextHttpImageUrl();
String fileName = selectedSource.getFileForHttpImage(imageRef);
if (imageRef != null && fileName != null) {
imageRef = selectedSource.getSourceName() + imageRef;
try {
card.setToken(selectedSource.isTokenSource());
Runnable task = new DownloadTask(card, imageRef, fileName, selectedSource.getTotalImages());
executor.execute(task);
} catch (Exception ex) {
}
} else if (selectedSource.getTotalImages() == -1) {
logger.info("Image not available on " + selectedSource.getSourceName() + ": " + card.getName() + " (" + card.getSet() + ')');
synchronized (sync) {
update(cardIndex + 1, cardsDownloadQueue.size());
CardImageUrls urls;
if (card.isToken()) {
if (!"0".equals(card.getCollectorId())) {
continue;
}
urls = selectedSource.generateTokenUrl(card);
} else {
urls = selectedSource.generateCardUrl(card);
}
} else {
Runnable task = new DownloadTask(card, urls, cardsDownloadQueue.size());
executor.execute(task);
if (urls == null) {
String imageRef = selectedSource.getNextHttpImageUrl();
String fileName = selectedSource.getFileForHttpImage(imageRef);
if (imageRef != null && fileName != null) {
imageRef = selectedSource.getSourceName() + imageRef;
try {
card.setToken(selectedSource.isTokenSource());
Runnable task = new DownloadTask(card, imageRef, fileName, selectedSource.getTotalImages());
executor.execute(task);
} catch (Exception ex) {
}
} else if (selectedSource.getTotalImages() == -1) {
logger.info("Image not available on " + selectedSource.getSourceName() + ": " + card.getName() + " (" + card.getSet() + ')');
synchronized (sync) {
update(cardIndex + 1, cardsDownloadQueue.size());
}
}
} else {
Runnable task = new DownloadTask(card, urls, cardsDownloadQueue.size());
executor.execute(task);
}
} catch (Exception ex) {
logger.error(ex, ex);
}
} catch (Exception ex) {
logger.error(ex, ex);
}
}
executor.shutdown();
while (!executor.isTerminated()) {
try {
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException ie) {
executor.shutdown();
while (!executor.isTerminated()) {
try {
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException ie) {
}
}
}
}
@ -671,6 +675,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
// stop
reloadCardsToDownload(uiDialog.getSetsCombo().getSelectedItem().toString());
enableDialogButtons();
// reset images cache
ImageCache.clearCache();
@ -707,7 +712,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
@Override
public void run() {
if (DownloadPicturesService.getInstance().getNeedCancel()) {
if (DownloadPicturesService.getInstance().isNeedCancel()) {
synchronized (sync) {
update(cardIndex + 1, count);
}
@ -791,14 +796,14 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
URL url = new URL(currentUrl);
// on download cancel need to stop
if (DownloadPicturesService.getInstance().getNeedCancel()) {
if (DownloadPicturesService.getInstance().isNeedCancel()) {
return;
}
// download
selectedSource.doPause(url.getPath());
httpConn = url.openConnection(p);
httpConn = url.openConnection(proxy);
if (httpConn != null) {
httpConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9.2.2) Gecko/20100316 Firefox/3.6.2");
@ -847,7 +852,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
int len;
while ((len = in.read(buf)) != -1) {
// user cancelled
if (DownloadPicturesService.getInstance().getNeedCancel()) {
if (DownloadPicturesService.getInstance().isNeedCancel()) {
// stop download, save current state and exit
TFile archive = destFile.getTopLevelArchive();
///* not need to unmout/close - it's auto action
@ -934,14 +939,21 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
// try download again
}
this.uiDialog.getRedownloadCheckbox().setSelected(false);
uiDialog.enableActionControls(true);
uiDialog.getStartButton().setEnabled(true);
enableDialogButtons();
}
}
private static final long serialVersionUID = 1L;
private void enableDialogButtons() {
uiDialog.getRedownloadCheckbox().setSelected(false); // reset re-download button after finished
uiDialog.enableActionControls(true);
uiDialog.getStartButton().setEnabled(true);
}
public Proxy getProxy() {
return proxy;
}
}
class LoadMissingCardDataNew implements Runnable {