mirror of
https://github.com/correl/mage.git
synced 2024-11-15 03:00:16 +00:00
* Fixed svg symbols download from scryfall (#5135);
This commit is contained in:
Oleg Agafonov
parent
6dce0c26f3
commit
69465fd9b6
6 changed files with 140 additions and 60 deletions
|
|
@ -158,6 +158,7 @@ public class DownloadGui extends JPanel {
|
|||
b.addActionListener(e -> {
|
||||
switch(this.job.getState()) {
|
||||
case NEW:
|
||||
case PREPARING:
|
||||
case WORKING:
|
||||
this.job.setState(State.ABORTED);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,16 +23,16 @@ import org.mage.plugins.card.utils.CardImageUtils;
|
|||
* The class DownloadJob.
|
||||
*
|
||||
* @version V0.0 25.08.2010
|
||||
* @author Clemens Koza
|
||||
* @author Clemens Koza, JayDi85
|
||||
*/
|
||||
public class DownloadJob extends AbstractLaternaBean {
|
||||
|
||||
public enum State {
|
||||
NEW, WORKING, FINISHED, ABORTED
|
||||
NEW, PREPARING, WORKING, FINISHED, ABORTED
|
||||
}
|
||||
|
||||
private final String name;
|
||||
private final Source source;
|
||||
private Source source;
|
||||
private final Destination destination;
|
||||
private final Property<State> state = properties.property("state", State.NEW);
|
||||
private final Property<String> message = properties.property("message");
|
||||
|
|
@ -98,6 +98,36 @@ public class DownloadJob extends AbstractLaternaBean {
|
|||
this.message.setValue(message);
|
||||
}
|
||||
|
||||
/**
|
||||
* Inner prepare cycle from new to working
|
||||
*/
|
||||
public void doPrepareAndStartWork() {
|
||||
if (this.state.getValue() != State.NEW) {
|
||||
setError("Can't call prepare at this point.");
|
||||
return;
|
||||
}
|
||||
|
||||
this.state.setValue(State.PREPARING);
|
||||
|
||||
try {
|
||||
onPreparing();
|
||||
} catch (Exception e) {
|
||||
setError("Prepare error: " + e.getMessage(), e);
|
||||
return;
|
||||
}
|
||||
|
||||
// change to working state on good prepare call
|
||||
this.state.setValue(State.WORKING);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Prepare code to override in custom download tasks (it's calls before work start)
|
||||
*/
|
||||
public void onPreparing() throws Exception {
|
||||
return;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the job's message.
|
||||
*
|
||||
|
|
@ -131,6 +161,10 @@ public class DownloadJob extends AbstractLaternaBean {
|
|||
return source;
|
||||
}
|
||||
|
||||
public void setSource(Source source) {
|
||||
this.source = source;
|
||||
}
|
||||
|
||||
public Destination getDestination() {
|
||||
return destination;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ public class Downloader extends AbstractLaternaBean implements Disposable {
|
|||
for (DownloadJob j : jobs) {
|
||||
switch (j.getState()) {
|
||||
case NEW:
|
||||
case PREPARING:
|
||||
case WORKING:
|
||||
j.setState(State.ABORTED);
|
||||
}
|
||||
|
|
@ -82,8 +83,11 @@ public class Downloader extends AbstractLaternaBean implements Disposable {
|
|||
}
|
||||
|
||||
public void add(DownloadJob job) {
|
||||
if (job.getState() == State.PREPARING) {
|
||||
throw new IllegalArgumentException("Job already preparing");
|
||||
}
|
||||
if (job.getState() == State.WORKING) {
|
||||
throw new IllegalArgumentException("Job already running");
|
||||
throw new IllegalArgumentException("Job already working");
|
||||
}
|
||||
if (job.getState() == State.FINISHED) {
|
||||
throw new IllegalArgumentException("Job already finished");
|
||||
|
|
@ -106,13 +110,22 @@ public class Downloader extends AbstractLaternaBean implements Disposable {
|
|||
|
||||
@Override
|
||||
public void onMessage(DownloadJob job) {
|
||||
//the job won't be processed by multiple threads
|
||||
|
||||
// start to work
|
||||
// the job won't be processed by multiple threads
|
||||
synchronized (job) {
|
||||
if (job.getState() != State.NEW) {
|
||||
return;
|
||||
}
|
||||
job.setState(State.WORKING);
|
||||
|
||||
job.doPrepareAndStartWork();
|
||||
|
||||
if (job.getState() != State.WORKING) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// download and save data
|
||||
try {
|
||||
Source src = job.getSource();
|
||||
Destination dst = job.getDestination();
|
||||
|
|
|
|||
|
|
@ -9,8 +9,13 @@ import java.util.*;
|
|||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import mage.MageException;
|
||||
import mage.util.StreamUtils;
|
||||
import org.jsoup.select.Elements;
|
||||
import org.mage.plugins.card.dl.DownloadJob;
|
||||
import org.mage.plugins.card.utils.CardImageUtils;
|
||||
|
||||
import javax.swing.text.Document;
|
||||
|
||||
import static org.mage.card.arcane.ManaSymbols.getSymbolFileNameAsSVG;
|
||||
import static org.mage.plugins.card.utils.CardImageUtils.getImagesDir;
|
||||
|
|
@ -18,14 +23,14 @@ import static org.mage.plugins.card.utils.CardImageUtils.getImagesDir;
|
|||
// TODO: add force to download symbols (rewrite exist files)
|
||||
|
||||
/**
|
||||
*
|
||||
* @author jaydi85@gmail.com
|
||||
*
|
||||
* @author JayDi85
|
||||
*/
|
||||
public class ScryfallSymbolsSource implements Iterable<DownloadJob> {
|
||||
|
||||
|
||||
static final String SOURCE_URL = "https://assets.scryfall.com/assets/scryfall.css"; // search css-file on https://scryfall.com/docs/api/colors
|
||||
//static final String SOURCE_URL = "https://assets.scryfall.com/assets/scryfall.css"; // old version with direct css-file on https://scryfall.com/docs/api/colors
|
||||
static final String CSS_SOURCE_URL = "https://scryfall.com/docs/api/colors";
|
||||
static final String CSS_SOURCE_SELECTOR = "link[rel=stylesheet]"; // <link rel="stylesheet" media="all" href="https://assets.scryfall.com/assets/scryfall-8cdaa786c4c86d5c49317e3e0fed72a0e409666753d3d3e8c906f33a188e19ed.css">
|
||||
static final String STATE_PROP_NAME = "state";
|
||||
static final String DOWNLOAD_TEMP_FILE = getImagesDir() + File.separator + "temp" + File.separator + "scryfall-symbols-source.txt";
|
||||
|
||||
|
|
@ -55,21 +60,21 @@ public class ScryfallSymbolsSource implements Iterable<DownloadJob> {
|
|||
return jobs.iterator();
|
||||
}
|
||||
|
||||
private void parseData(String sourcePath){
|
||||
private void parseData(String sourcePath) {
|
||||
|
||||
String sourceData = "";
|
||||
try {
|
||||
sourceData = new String(Files.readAllBytes(Paths.get(sourcePath)));
|
||||
}catch (IOException e) {
|
||||
} catch (IOException e) {
|
||||
LOGGER.error("Can't open file to parse data: " + sourcePath + " , reason: " + e.getMessage());
|
||||
}
|
||||
|
||||
// gen symbols list
|
||||
ArrayList<String> allMageSymbols = new ArrayList<>();
|
||||
for(int i = 0; i < SYMBOLS_LIST.length; i++){
|
||||
for (int i = 0; i < SYMBOLS_LIST.length; i++) {
|
||||
allMageSymbols.add(SYMBOLS_LIST[i]);
|
||||
}
|
||||
for(Integer i = SYMBOLS_NUMBER_START; i <= SYMBOLS_NUMBER_END; i++){
|
||||
for (Integer i = SYMBOLS_NUMBER_START; i <= SYMBOLS_NUMBER_END; i++) {
|
||||
allMageSymbols.add(String.valueOf(SYMBOLS_NUMBER_START + i));
|
||||
}
|
||||
|
||||
|
|
@ -88,23 +93,22 @@ public class ScryfallSymbolsSource implements Iterable<DownloadJob> {
|
|||
|
||||
// dirs maker
|
||||
File dir = getSymbolFileNameAsSVG("W").getParentFile();
|
||||
if(!dir.exists()){
|
||||
if (!dir.exists()) {
|
||||
dir.mkdirs();
|
||||
}
|
||||
|
||||
// decode and save data (only if not exist)
|
||||
for(String needCode: allMageSymbols){
|
||||
for (String needCode : allMageSymbols) {
|
||||
|
||||
String searchCode = needCode.replace("/", "");
|
||||
|
||||
if(!foundedData.containsKey(searchCode))
|
||||
{
|
||||
if (!foundedData.containsKey(searchCode)) {
|
||||
LOGGER.warn("Can't found symbol code from scryfall: " + searchCode);
|
||||
continue;
|
||||
}
|
||||
|
||||
File destFile = getSymbolFileNameAsSVG(searchCode);
|
||||
if (destFile.exists() && (destFile.length() > 0)){
|
||||
if (destFile.exists() && (destFile.length() > 0)) {
|
||||
continue;
|
||||
}
|
||||
FileOutputStream stream = null;
|
||||
|
|
@ -118,7 +122,7 @@ public class ScryfallSymbolsSource implements Iterable<DownloadJob> {
|
|||
stream.write(fileData);
|
||||
|
||||
LOGGER.info("New svg symbol downloaded: " + needCode);
|
||||
} catch (Exception e) {
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("Can't decode svg icon and save to file: " + destFile.getPath() + ", reason: " + e.getMessage());
|
||||
} finally {
|
||||
StreamUtils.closeQuietly(stream);
|
||||
|
|
@ -127,23 +131,26 @@ public class ScryfallSymbolsSource implements Iterable<DownloadJob> {
|
|||
}
|
||||
|
||||
|
||||
private class ScryfallSymbolsDownloadJob extends DownloadJob{
|
||||
private class ScryfallSymbolsDownloadJob extends DownloadJob {
|
||||
|
||||
private String cssUrl = ""; // need to find url from colors page https://scryfall.com/docs/api/colors
|
||||
|
||||
// listener for data parse after download complete
|
||||
private class ScryDownloadOnFinishedListener implements PropertyChangeListener {
|
||||
private class ScryfallDownloadOnFinishedListener implements PropertyChangeListener {
|
||||
private String downloadedFile;
|
||||
|
||||
public ScryDownloadOnFinishedListener(String ADestFile){
|
||||
public ScryfallDownloadOnFinishedListener(String ADestFile) {
|
||||
this.downloadedFile = ADestFile;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void propertyChange(PropertyChangeEvent evt) {
|
||||
if (!evt.getPropertyName().equals(STATE_PROP_NAME)){
|
||||
|
||||
if (!evt.getPropertyName().equals(STATE_PROP_NAME)) {
|
||||
throw new IllegalArgumentException("Unknown download property " + evt.getPropertyName());
|
||||
}
|
||||
|
||||
if (evt.getNewValue() != State.FINISHED){
|
||||
if (evt.getNewValue() != State.FINISHED) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -152,16 +159,34 @@ public class ScryfallSymbolsSource implements Iterable<DownloadJob> {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onPreparing() throws Exception {
|
||||
this.cssUrl = "";
|
||||
|
||||
org.jsoup.nodes.Document doc = CardImageUtils.downloadHtmlDocument(CSS_SOURCE_URL);
|
||||
org.jsoup.select.Elements cssList = doc.select(CSS_SOURCE_SELECTOR);
|
||||
if (cssList.size() == 1) {
|
||||
this.cssUrl = cssList.first().attr("href").toString();
|
||||
}
|
||||
|
||||
if (this.cssUrl.isEmpty()) {
|
||||
throw new IllegalStateException("Can't find stylesheet url from scryfall colors page.");
|
||||
} else {
|
||||
this.setSource(fromURL(this.cssUrl));
|
||||
}
|
||||
}
|
||||
|
||||
private String destFile = "";
|
||||
|
||||
public ScryfallSymbolsDownloadJob() {
|
||||
super("Scryfall symbols source", fromURL(SOURCE_URL), toFile(DOWNLOAD_TEMP_FILE));
|
||||
// download init
|
||||
super("Scryfall symbols source", fromURL(""), toFile(DOWNLOAD_TEMP_FILE)); // url setup on preparing stage
|
||||
this.destFile = DOWNLOAD_TEMP_FILE;
|
||||
this.addPropertyChangeListener(STATE_PROP_NAME, new ScryDownloadOnFinishedListener(this.destFile));
|
||||
this.addPropertyChangeListener(STATE_PROP_NAME, new ScryfallDownloadOnFinishedListener(this.destFile));
|
||||
|
||||
// clear dest file (always download new data)
|
||||
File file = new File(this.destFile);
|
||||
if (file.exists()){
|
||||
if (file.exists()) {
|
||||
file.delete();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ import org.jsoup.nodes.Document;
|
|||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
import org.mage.plugins.card.images.CardDownloadData;
|
||||
import org.mage.plugins.card.utils.CardImageUtils;
|
||||
|
||||
/**
|
||||
* @author North
|
||||
|
|
@ -535,7 +536,7 @@ public enum WizardCardsImageSource implements CardImageSource {
|
|||
while (page < 999) {
|
||||
String searchUrl = "http://gatherer.wizards.com/Pages/Search/Default.aspx?sort=cn+&page=" + page + "&action=advanced&output=spoiler&method=visual&set=+%5B%22" + URLSetName + "%22%5D";
|
||||
logger.debug("URL: " + searchUrl);
|
||||
Document doc = getDocument(searchUrl);
|
||||
Document doc = CardImageUtils.downloadHtmlDocument(searchUrl);
|
||||
Elements cardsImages = doc.select("img[src^=../../Handlers/]");
|
||||
if (cardsImages.isEmpty()) {
|
||||
break;
|
||||
|
|
@ -587,33 +588,6 @@ public enum WizardCardsImageSource implements CardImageSource {
|
|||
return setLinks;
|
||||
}
|
||||
|
||||
private Document getDocument(String urlString) throws NumberFormatException, IOException {
|
||||
Preferences prefs = MageFrame.getPreferences();
|
||||
Connection.ProxyType proxyType = Connection.ProxyType.valueByText(prefs.get("proxyType", "None"));
|
||||
Document doc;
|
||||
if (proxyType == ProxyType.NONE) {
|
||||
doc = Jsoup.connect(urlString).timeout(60 * 1000).get();
|
||||
} else {
|
||||
String proxyServer = prefs.get("proxyAddress", "");
|
||||
int proxyPort = Integer.parseInt(prefs.get("proxyPort", "0"));
|
||||
URL url = new URL(urlString);
|
||||
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyServer, proxyPort));
|
||||
HttpURLConnection uc = (HttpURLConnection) url.openConnection(proxy);
|
||||
uc.setConnectTimeout(10000);
|
||||
uc.setReadTimeout(60000);
|
||||
uc.connect();
|
||||
|
||||
String line;
|
||||
StringBuffer tmp = new StringBuffer();
|
||||
BufferedReader in = new BufferedReader(new InputStreamReader(uc.getInputStream()));
|
||||
while ((line = in.readLine()) != null) {
|
||||
tmp.append(line);
|
||||
}
|
||||
doc = Jsoup.parse(String.valueOf(tmp));
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
private void getLandVariations(LinkedHashMap<String, String> setLinks, String cardSet, int multiverseId, String cardName) throws IOException, NumberFormatException {
|
||||
CardCriteria criteria = new CardCriteria();
|
||||
criteria.nameExact(cardName);
|
||||
|
|
@ -621,7 +595,7 @@ public enum WizardCardsImageSource implements CardImageSource {
|
|||
List<CardInfo> cards = CardRepository.instance.findCards(criteria);
|
||||
|
||||
String urlLandDocument = "http://gatherer.wizards.com/Pages/Card/Details.aspx?multiverseid=" + multiverseId;
|
||||
Document landDoc = getDocument(urlLandDocument);
|
||||
Document landDoc = CardImageUtils.downloadHtmlDocument(urlLandDocument);
|
||||
Elements variations = landDoc.select("a.variationlink");
|
||||
if (!variations.isEmpty()) {
|
||||
if (variations.size() > cards.size()) {
|
||||
|
|
@ -668,7 +642,7 @@ public enum WizardCardsImageSource implements CardImageSource {
|
|||
|
||||
private HashMap<String, Integer> getlocalizedMultiverseIds(Integer englishMultiverseId) throws IOException {
|
||||
String cardLanguagesUrl = "http://gatherer.wizards.com/Pages/Card/Languages.aspx?multiverseid=" + englishMultiverseId;
|
||||
Document cardLanguagesDoc = getDocument(cardLanguagesUrl);
|
||||
Document cardLanguagesDoc = CardImageUtils.downloadHtmlDocument(cardLanguagesUrl);
|
||||
Elements languageTableRows = cardLanguagesDoc.select("tr.cardItem");
|
||||
HashMap<String, Integer> localizedIds = new HashMap<>();
|
||||
if (!languageTableRows.isEmpty()) {
|
||||
|
|
|
|||
|
|
@ -1,11 +1,17 @@
|
|||
package org.mage.plugins.card.utils;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.Proxy;
|
||||
import java.net.URL;
|
||||
import java.util.HashMap;
|
||||
import java.util.Locale;
|
||||
import java.util.prefs.Preferences;
|
||||
|
||||
import mage.client.MageFrame;
|
||||
import mage.client.constants.Constants;
|
||||
import mage.client.dialog.PreferencesDialog;
|
||||
|
|
@ -13,6 +19,8 @@ import mage.remote.Connection;
|
|||
import mage.remote.Connection.ProxyType;
|
||||
import net.java.truevfs.access.TFile;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.mage.plugins.card.images.CardDownloadData;
|
||||
import org.mage.plugins.card.properties.SettingsManager;
|
||||
|
||||
|
|
@ -22,7 +30,6 @@ public final class CardImageUtils {
|
|||
private static final Logger log = Logger.getLogger(CardImageUtils.class);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param card
|
||||
* @return String if image exists, else null
|
||||
*/
|
||||
|
|
@ -54,7 +61,6 @@ public final class CardImageUtils {
|
|||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param card
|
||||
* @return String regardless of whether image exists
|
||||
*/
|
||||
|
|
@ -280,4 +286,31 @@ public final class CardImageUtils {
|
|||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static Document downloadHtmlDocument(String urlString) throws NumberFormatException, IOException {
|
||||
Preferences prefs = MageFrame.getPreferences();
|
||||
Connection.ProxyType proxyType = Connection.ProxyType.valueByText(prefs.get("proxyType", "None"));
|
||||
Document doc;
|
||||
if (proxyType == ProxyType.NONE) {
|
||||
doc = Jsoup.connect(urlString).timeout(60 * 1000).get();
|
||||
} else {
|
||||
String proxyServer = prefs.get("proxyAddress", "");
|
||||
int proxyPort = Integer.parseInt(prefs.get("proxyPort", "0"));
|
||||
URL url = new URL(urlString);
|
||||
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyServer, proxyPort));
|
||||
HttpURLConnection uc = (HttpURLConnection) url.openConnection(proxy);
|
||||
uc.setConnectTimeout(10000);
|
||||
uc.setReadTimeout(60000);
|
||||
uc.connect();
|
||||
|
||||
String line;
|
||||
StringBuffer tmp = new StringBuffer();
|
||||
BufferedReader in = new BufferedReader(new InputStreamReader(uc.getInputStream()));
|
||||
while ((line = in.readLine()) != null) {
|
||||
tmp.append(line);
|
||||
}
|
||||
doc = Jsoup.parse(String.valueOf(tmp));
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue