github/mage
1
0
Fork 0
mirror of https://github.com/correl/mage.git synced 2025-04-10 01:01:05 -09:00
Code Issues Projects Releases Packages Wiki Activity

* Fixed svg symbols download from scryfall ();

Browse source
This commit is contained in:
Oleg Agafonov 2018-07-10 05:49:21 +04:00
parent 6dce0c26f3
commit 69465fd9b6
6 changed files with 140 additions and 60 deletions

1
Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadGui.java
View file

@ -158,6 +158,7 @@ public class DownloadGui extends JPanel {
b.addActionListener(e -> { b.addActionListener(e -> {
switch(this.job.getState()) { switch(this.job.getState()) {
case NEW: case NEW:
case PREPARING:
case WORKING: case WORKING:
this.job.setState(State.ABORTED); this.job.setState(State.ABORTED);
} }

40
Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadJob.java
View file

@ -23,16 +23,16 @@ import org.mage.plugins.card.utils.CardImageUtils;
* The class DownloadJob. * The class DownloadJob.
* *
* @version V0.0 25.08.2010 * @version V0.0 25.08.2010
* @author Clemens Koza * @author Clemens Koza, JayDi85
*/ */
public class DownloadJob extends AbstractLaternaBean { public class DownloadJob extends AbstractLaternaBean {
public enum State { public enum State {
NEW, WORKING, FINISHED, ABORTED NEW, PREPARING, WORKING, FINISHED, ABORTED
} }
private final String name; private final String name;
private final Source source; private Source source;
private final Destination destination; private final Destination destination;
private final Property<State> state = properties.property("state", State.NEW); private final Property<State> state = properties.property("state", State.NEW);
private final Property<String> message = properties.property("message"); private final Property<String> message = properties.property("message");
@ -98,6 +98,36 @@ public class DownloadJob extends AbstractLaternaBean {
this.message.setValue(message); this.message.setValue(message);
} }
/**
* Inner prepare cycle from new to working
*/
public void doPrepareAndStartWork() {
if (this.state.getValue() != State.NEW) {
setError("Can't call prepare at this point.");
return;
}
this.state.setValue(State.PREPARING);
try {
onPreparing();
} catch (Exception e) {
setError("Prepare error: " + e.getMessage(), e);
return;
}
// change to working state on good prepare call
this.state.setValue(State.WORKING);
}
/**
* Prepare code to override in custom download tasks (it's calls before work start)
*/
public void onPreparing() throws Exception {
return;
}
/** /**
* Sets the job's message. * Sets the job's message.
* *
@ -131,6 +161,10 @@ public class DownloadJob extends AbstractLaternaBean {
return source; return source;
} }
public void setSource(Source source) {
this.source = source;
}
public Destination getDestination() { public Destination getDestination() {
return destination; return destination;
} }

19
Mage.Client/src/main/java/org/mage/plugins/card/dl/Downloader.java
View file

@ -60,6 +60,7 @@ public class Downloader extends AbstractLaternaBean implements Disposable {
for (DownloadJob j : jobs) { for (DownloadJob j : jobs) {
switch (j.getState()) { switch (j.getState()) {
case NEW: case NEW:
case PREPARING:
case WORKING: case WORKING:
j.setState(State.ABORTED); j.setState(State.ABORTED);
} }
@ -82,8 +83,11 @@ public class Downloader extends AbstractLaternaBean implements Disposable {
} }
public void add(DownloadJob job) { public void add(DownloadJob job) {
if (job.getState() == State.PREPARING) {
throw new IllegalArgumentException("Job already preparing");
}
if (job.getState() == State.WORKING) { if (job.getState() == State.WORKING) {
throw new IllegalArgumentException("Job already running"); throw new IllegalArgumentException("Job already working");
} }
if (job.getState() == State.FINISHED) { if (job.getState() == State.FINISHED) {
throw new IllegalArgumentException("Job already finished"); throw new IllegalArgumentException("Job already finished");
@ -106,13 +110,22 @@ public class Downloader extends AbstractLaternaBean implements Disposable {
@Override @Override
public void onMessage(DownloadJob job) { public void onMessage(DownloadJob job) {
//the job won't be processed by multiple threads
// start to work
// the job won't be processed by multiple threads
synchronized (job) { synchronized (job) {
if (job.getState() != State.NEW) { if (job.getState() != State.NEW) {
return; return;
} }
job.setState(State.WORKING);
job.doPrepareAndStartWork();
if (job.getState() != State.WORKING) {
return;
}
} }
// download and save data
try { try {
Source src = job.getSource(); Source src = job.getSource();
Destination dst = job.getDestination(); Destination dst = job.getDestination();

69
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallSymbolsSource.java
View file

@ -9,8 +9,13 @@ import java.util.*;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import mage.MageException;
import mage.util.StreamUtils; import mage.util.StreamUtils;
import org.jsoup.select.Elements;
import org.mage.plugins.card.dl.DownloadJob; import org.mage.plugins.card.dl.DownloadJob;
import org.mage.plugins.card.utils.CardImageUtils;
import javax.swing.text.Document;
import static org.mage.card.arcane.ManaSymbols.getSymbolFileNameAsSVG; import static org.mage.card.arcane.ManaSymbols.getSymbolFileNameAsSVG;
import static org.mage.plugins.card.utils.CardImageUtils.getImagesDir; import static org.mage.plugins.card.utils.CardImageUtils.getImagesDir;
@ -18,14 +23,14 @@ import static org.mage.plugins.card.utils.CardImageUtils.getImagesDir;
// TODO: add force to download symbols (rewrite exist files) // TODO: add force to download symbols (rewrite exist files)
/** /**
* * @author JayDi85
* @author jaydi85@gmail.com
*
*/ */
public class ScryfallSymbolsSource implements Iterable<DownloadJob> { public class ScryfallSymbolsSource implements Iterable<DownloadJob> {
static final String SOURCE_URL = "https://assets.scryfall.com/assets/scryfall.css"; // search css-file on https://scryfall.com/docs/api/colors //static final String SOURCE_URL = "https://assets.scryfall.com/assets/scryfall.css"; // old version with direct css-file on https://scryfall.com/docs/api/colors
static final String CSS_SOURCE_URL = "https://scryfall.com/docs/api/colors";
static final String CSS_SOURCE_SELECTOR = "link[rel=stylesheet]"; // <link rel="stylesheet" media="all" href="https://assets.scryfall.com/assets/scryfall-8cdaa786c4c86d5c49317e3e0fed72a0e409666753d3d3e8c906f33a188e19ed.css">
static final String STATE_PROP_NAME = "state"; static final String STATE_PROP_NAME = "state";
static final String DOWNLOAD_TEMP_FILE = getImagesDir() + File.separator + "temp" + File.separator + "scryfall-symbols-source.txt"; static final String DOWNLOAD_TEMP_FILE = getImagesDir() + File.separator + "temp" + File.separator + "scryfall-symbols-source.txt";
@ -55,21 +60,21 @@ public class ScryfallSymbolsSource implements Iterable<DownloadJob> {
return jobs.iterator(); return jobs.iterator();
} }
private void parseData(String sourcePath){ private void parseData(String sourcePath) {
String sourceData = ""; String sourceData = "";
try { try {
sourceData = new String(Files.readAllBytes(Paths.get(sourcePath))); sourceData = new String(Files.readAllBytes(Paths.get(sourcePath)));
}catch (IOException e) { } catch (IOException e) {
LOGGER.error("Can't open file to parse data: " + sourcePath + " , reason: " + e.getMessage()); LOGGER.error("Can't open file to parse data: " + sourcePath + " , reason: " + e.getMessage());
} }
// gen symbols list // gen symbols list
ArrayList<String> allMageSymbols = new ArrayList<>(); ArrayList<String> allMageSymbols = new ArrayList<>();
for(int i = 0; i < SYMBOLS_LIST.length; i++){ for (int i = 0; i < SYMBOLS_LIST.length; i++) {
allMageSymbols.add(SYMBOLS_LIST[i]); allMageSymbols.add(SYMBOLS_LIST[i]);
} }
for(Integer i = SYMBOLS_NUMBER_START; i <= SYMBOLS_NUMBER_END; i++){ for (Integer i = SYMBOLS_NUMBER_START; i <= SYMBOLS_NUMBER_END; i++) {
allMageSymbols.add(String.valueOf(SYMBOLS_NUMBER_START + i)); allMageSymbols.add(String.valueOf(SYMBOLS_NUMBER_START + i));
} }
@ -88,23 +93,22 @@ public class ScryfallSymbolsSource implements Iterable<DownloadJob> {
// dirs maker // dirs maker
File dir = getSymbolFileNameAsSVG("W").getParentFile(); File dir = getSymbolFileNameAsSVG("W").getParentFile();
if(!dir.exists()){ if (!dir.exists()) {
dir.mkdirs(); dir.mkdirs();
} }
// decode and save data (only if not exist) // decode and save data (only if not exist)
for(String needCode: allMageSymbols){ for (String needCode : allMageSymbols) {
String searchCode = needCode.replace("/", ""); String searchCode = needCode.replace("/", "");
if(!foundedData.containsKey(searchCode)) if (!foundedData.containsKey(searchCode)) {
{
LOGGER.warn("Can't found symbol code from scryfall: " + searchCode); LOGGER.warn("Can't found symbol code from scryfall: " + searchCode);
continue; continue;
} }
File destFile = getSymbolFileNameAsSVG(searchCode); File destFile = getSymbolFileNameAsSVG(searchCode);
if (destFile.exists() && (destFile.length() > 0)){ if (destFile.exists() && (destFile.length() > 0)) {
continue; continue;
} }
FileOutputStream stream = null; FileOutputStream stream = null;
@ -118,7 +122,7 @@ public class ScryfallSymbolsSource implements Iterable<DownloadJob> {
stream.write(fileData); stream.write(fileData);
LOGGER.info("New svg symbol downloaded: " + needCode); LOGGER.info("New svg symbol downloaded: " + needCode);
} catch (Exception e) { } catch (Exception e) {
LOGGER.error("Can't decode svg icon and save to file: " + destFile.getPath() + ", reason: " + e.getMessage()); LOGGER.error("Can't decode svg icon and save to file: " + destFile.getPath() + ", reason: " + e.getMessage());
} finally { } finally {
StreamUtils.closeQuietly(stream); StreamUtils.closeQuietly(stream);
@ -127,23 +131,26 @@ public class ScryfallSymbolsSource implements Iterable<DownloadJob> {
} }
private class ScryfallSymbolsDownloadJob extends DownloadJob{ private class ScryfallSymbolsDownloadJob extends DownloadJob {
private String cssUrl = ""; // need to find url from colors page https://scryfall.com/docs/api/colors
// listener for data parse after download complete // listener for data parse after download complete
private class ScryDownloadOnFinishedListener implements PropertyChangeListener { private class ScryfallDownloadOnFinishedListener implements PropertyChangeListener {
private String downloadedFile; private String downloadedFile;
public ScryDownloadOnFinishedListener(String ADestFile){ public ScryfallDownloadOnFinishedListener(String ADestFile) {
this.downloadedFile = ADestFile; this.downloadedFile = ADestFile;
} }
@Override @Override
public void propertyChange(PropertyChangeEvent evt) { public void propertyChange(PropertyChangeEvent evt) {
if (!evt.getPropertyName().equals(STATE_PROP_NAME)){
if (!evt.getPropertyName().equals(STATE_PROP_NAME)) {
throw new IllegalArgumentException("Unknown download property " + evt.getPropertyName()); throw new IllegalArgumentException("Unknown download property " + evt.getPropertyName());
} }
if (evt.getNewValue() != State.FINISHED){ if (evt.getNewValue() != State.FINISHED) {
return; return;
} }
@ -152,16 +159,34 @@ public class ScryfallSymbolsSource implements Iterable<DownloadJob> {
} }
} }
@Override
public void onPreparing() throws Exception {
this.cssUrl = "";
org.jsoup.nodes.Document doc = CardImageUtils.downloadHtmlDocument(CSS_SOURCE_URL);
org.jsoup.select.Elements cssList = doc.select(CSS_SOURCE_SELECTOR);
if (cssList.size() == 1) {
this.cssUrl = cssList.first().attr("href").toString();
}
if (this.cssUrl.isEmpty()) {
throw new IllegalStateException("Can't find stylesheet url from scryfall colors page.");
} else {
this.setSource(fromURL(this.cssUrl));
}
}
private String destFile = ""; private String destFile = "";
public ScryfallSymbolsDownloadJob() { public ScryfallSymbolsDownloadJob() {
super("Scryfall symbols source", fromURL(SOURCE_URL), toFile(DOWNLOAD_TEMP_FILE)); // download init
super("Scryfall symbols source", fromURL(""), toFile(DOWNLOAD_TEMP_FILE)); // url setup on preparing stage
this.destFile = DOWNLOAD_TEMP_FILE; this.destFile = DOWNLOAD_TEMP_FILE;
this.addPropertyChangeListener(STATE_PROP_NAME, new ScryDownloadOnFinishedListener(this.destFile)); this.addPropertyChangeListener(STATE_PROP_NAME, new ScryfallDownloadOnFinishedListener(this.destFile));
// clear dest file (always download new data) // clear dest file (always download new data)
File file = new File(this.destFile); File file = new File(this.destFile);
if (file.exists()){ if (file.exists()) {
file.delete(); file.delete();
} }
} }

34
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/WizardCardsImageSource.java
View file

@ -35,6 +35,7 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.select.Elements; import org.jsoup.select.Elements;
import org.mage.plugins.card.images.CardDownloadData; import org.mage.plugins.card.images.CardDownloadData;
import org.mage.plugins.card.utils.CardImageUtils;
/** /**
* @author North * @author North
@ -535,7 +536,7 @@ public enum WizardCardsImageSource implements CardImageSource {
while (page < 999) { while (page < 999) {
String searchUrl = "http://gatherer.wizards.com/Pages/Search/Default.aspx?sort=cn+&page=" + page + "&action=advanced&output=spoiler&method=visual&set=+%5B%22" + URLSetName + "%22%5D"; String searchUrl = "http://gatherer.wizards.com/Pages/Search/Default.aspx?sort=cn+&page=" + page + "&action=advanced&output=spoiler&method=visual&set=+%5B%22" + URLSetName + "%22%5D";
logger.debug("URL: " + searchUrl); logger.debug("URL: " + searchUrl);
Document doc = getDocument(searchUrl); Document doc = CardImageUtils.downloadHtmlDocument(searchUrl);
Elements cardsImages = doc.select("img[src^=../../Handlers/]"); Elements cardsImages = doc.select("img[src^=../../Handlers/]");
if (cardsImages.isEmpty()) { if (cardsImages.isEmpty()) {
break; break;
@ -587,33 +588,6 @@ public enum WizardCardsImageSource implements CardImageSource {
return setLinks; return setLinks;
} }
private Document getDocument(String urlString) throws NumberFormatException, IOException {
Preferences prefs = MageFrame.getPreferences();
Connection.ProxyType proxyType = Connection.ProxyType.valueByText(prefs.get("proxyType", "None"));
Document doc;
if (proxyType == ProxyType.NONE) {
doc = Jsoup.connect(urlString).timeout(60 * 1000).get();
} else {
String proxyServer = prefs.get("proxyAddress", "");
int proxyPort = Integer.parseInt(prefs.get("proxyPort", "0"));
URL url = new URL(urlString);
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyServer, proxyPort));
HttpURLConnection uc = (HttpURLConnection) url.openConnection(proxy);
uc.setConnectTimeout(10000);
uc.setReadTimeout(60000);
uc.connect();
String line;
StringBuffer tmp = new StringBuffer();
BufferedReader in = new BufferedReader(new InputStreamReader(uc.getInputStream()));
while ((line = in.readLine()) != null) {
tmp.append(line);
}
doc = Jsoup.parse(String.valueOf(tmp));
}
return doc;
}
private void getLandVariations(LinkedHashMap<String, String> setLinks, String cardSet, int multiverseId, String cardName) throws IOException, NumberFormatException { private void getLandVariations(LinkedHashMap<String, String> setLinks, String cardSet, int multiverseId, String cardName) throws IOException, NumberFormatException {
CardCriteria criteria = new CardCriteria(); CardCriteria criteria = new CardCriteria();
criteria.nameExact(cardName); criteria.nameExact(cardName);
@ -621,7 +595,7 @@ public enum WizardCardsImageSource implements CardImageSource {
List<CardInfo> cards = CardRepository.instance.findCards(criteria); List<CardInfo> cards = CardRepository.instance.findCards(criteria);
String urlLandDocument = "http://gatherer.wizards.com/Pages/Card/Details.aspx?multiverseid=" + multiverseId; String urlLandDocument = "http://gatherer.wizards.com/Pages/Card/Details.aspx?multiverseid=" + multiverseId;
Document landDoc = getDocument(urlLandDocument); Document landDoc = CardImageUtils.downloadHtmlDocument(urlLandDocument);
Elements variations = landDoc.select("a.variationlink"); Elements variations = landDoc.select("a.variationlink");
if (!variations.isEmpty()) { if (!variations.isEmpty()) {
if (variations.size() > cards.size()) { if (variations.size() > cards.size()) {
@ -668,7 +642,7 @@ public enum WizardCardsImageSource implements CardImageSource {
private HashMap<String, Integer> getlocalizedMultiverseIds(Integer englishMultiverseId) throws IOException { private HashMap<String, Integer> getlocalizedMultiverseIds(Integer englishMultiverseId) throws IOException {
String cardLanguagesUrl = "http://gatherer.wizards.com/Pages/Card/Languages.aspx?multiverseid=" + englishMultiverseId; String cardLanguagesUrl = "http://gatherer.wizards.com/Pages/Card/Languages.aspx?multiverseid=" + englishMultiverseId;
Document cardLanguagesDoc = getDocument(cardLanguagesUrl); Document cardLanguagesDoc = CardImageUtils.downloadHtmlDocument(cardLanguagesUrl);
Elements languageTableRows = cardLanguagesDoc.select("tr.cardItem"); Elements languageTableRows = cardLanguagesDoc.select("tr.cardItem");
HashMap<String, Integer> localizedIds = new HashMap<>(); HashMap<String, Integer> localizedIds = new HashMap<>();
if (!languageTableRows.isEmpty()) { if (!languageTableRows.isEmpty()) {

37
Mage.Client/src/main/java/org/mage/plugins/card/utils/CardImageUtils.java
View file

@ -1,11 +1,17 @@
package org.mage.plugins.card.utils; package org.mage.plugins.card.utils;
import java.io.BufferedReader;
import java.io.File; import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
import java.net.Proxy; import java.net.Proxy;
import java.net.URL;
import java.util.HashMap; import java.util.HashMap;
import java.util.Locale; import java.util.Locale;
import java.util.prefs.Preferences; import java.util.prefs.Preferences;
import mage.client.MageFrame; import mage.client.MageFrame;
import mage.client.constants.Constants; import mage.client.constants.Constants;
import mage.client.dialog.PreferencesDialog; import mage.client.dialog.PreferencesDialog;
@ -13,6 +19,8 @@ import mage.remote.Connection;
import mage.remote.Connection.ProxyType; import mage.remote.Connection.ProxyType;
import net.java.truevfs.access.TFile; import net.java.truevfs.access.TFile;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.mage.plugins.card.images.CardDownloadData; import org.mage.plugins.card.images.CardDownloadData;
import org.mage.plugins.card.properties.SettingsManager; import org.mage.plugins.card.properties.SettingsManager;
@ -22,7 +30,6 @@ public final class CardImageUtils {
private static final Logger log = Logger.getLogger(CardImageUtils.class); private static final Logger log = Logger.getLogger(CardImageUtils.class);
/** /**
*
* @param card * @param card
* @return String if image exists, else null * @return String if image exists, else null
*/ */
@ -54,7 +61,6 @@ public final class CardImageUtils {
} }
/** /**
*
* @param card * @param card
* @return String regardless of whether image exists * @return String regardless of whether image exists
*/ */
@ -280,4 +286,31 @@ public final class CardImageUtils {
} }
return null; return null;
} }
public static Document downloadHtmlDocument(String urlString) throws NumberFormatException, IOException {
Preferences prefs = MageFrame.getPreferences();
Connection.ProxyType proxyType = Connection.ProxyType.valueByText(prefs.get("proxyType", "None"));
Document doc;
if (proxyType == ProxyType.NONE) {
doc = Jsoup.connect(urlString).timeout(60 * 1000).get();
} else {
String proxyServer = prefs.get("proxyAddress", "");
int proxyPort = Integer.parseInt(prefs.get("proxyPort", "0"));
URL url = new URL(urlString);
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyServer, proxyPort));
HttpURLConnection uc = (HttpURLConnection) url.openConnection(proxy);
uc.setConnectTimeout(10000);
uc.setReadTimeout(60000);
uc.connect();
String line;
StringBuffer tmp = new StringBuffer();
BufferedReader in = new BufferedReader(new InputStreamReader(uc.getInputStream()));
while ((line = in.readLine()) != null) {
tmp.append(line);
}
doc = Jsoup.parse(String.valueOf(tmp));
}
return doc;
}
} }