Merge pull request #4159 from JayDi85/master

Downloading images fixes (scryfall, wizards, symbols)

Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/DirectLinksForDownload.java

@@ -22,7 +22,7 @@ import static org.mage.plugins.card.dl.DownloadJob.toFile;
  */
 public class DirectLinksForDownload implements Iterable<DownloadJob> {
 
-    private static final String backsideUrl = "http://upload.wikimedia.org/wikipedia/en/a/aa/Magic_the_gathering-card_back.jpg";
+    private static final String backsideUrl = "https://upload.wikimedia.org/wikipedia/en/a/aa/Magic_the_gathering-card_back.jpg";
 
     private static final Map<String, String> directLinks = new LinkedHashMap<>();
 

Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/GathererSets.java

@@ -6,19 +6,44 @@ import java.util.Calendar;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.Iterator;
+
 import mage.cards.ExpansionSet;
 import mage.cards.Sets;
+import mage.constants.Rarity;
 import org.mage.plugins.card.dl.DownloadJob;
+
 import static org.mage.plugins.card.dl.DownloadJob.fromURL;
 import static org.mage.plugins.card.dl.DownloadJob.toFile;
+import org.apache.log4j.Logger;
 
 public class GathererSets implements Iterable<DownloadJob> {
 
+    private class CheckResult {
+        String code;
+        ExpansionSet set;
+        boolean haveCommon;
+        boolean haveUncommon;
+        boolean haveRare;
+        boolean haveMyth;
+
+        private CheckResult(String ACode, ExpansionSet ASet, boolean AHaveCommon, boolean AHaveUncommon, boolean AHhaveRare, boolean AHaveMyth) {
+            code = ACode;
+            set = ASet;
+            haveCommon = AHaveCommon;
+            haveUncommon = AHaveUncommon;
+            haveRare = AHhaveRare;
+            haveMyth = AHaveMyth;
+        }
+    }
+
+    private static final int DAYS_BEFORE_RELEASE_TO_DOWNLOAD = +14; // Try to load the symbolsBasic eralies 14 days before release date
+    private static final Logger logger = Logger.getLogger(GathererSets.class);
+
     private static final String SETS_PATH = File.separator + "sets";
     private static final File DEFAULT_OUT_DIR = new File("plugins" + File.separator + "images" + SETS_PATH);
     private static File outDir = DEFAULT_OUT_DIR;
 
-    private static final String[] symbols = {"10E", "9ED", "8ED", "7ED", "6ED", "5ED", "4ED", "3ED", "2ED", "LEB", "LEA",
+    private static final String[] symbolsBasic = {"10E", "9ED", "8ED", "7ED", "6ED", "5ED", "4ED", "3ED", "2ED", "LEB", "LEA",
         "HOP",
         "ARN", "ATQ", "LEG", "DRK", "FEM", "HML",
         "ICE", "ALL", "CSP",
@@ -36,14 +61,19 @@ public class GathererSets implements Iterable<DownloadJob> {
         "LRW", "MOR",
         "SHM", "EVE",
         "MED", "ME2", "ME3", "ME4",
-        "POR", "PO2", "PTK",
+        "POR", "P02", "PTK",
         "ARC", "DD3EVG",
-        "W16", "W17"};
+        "W16", "W17",
+        //"APAC" -- gatherer do not have that set, scrly have PALP
+        //"ARENA" -- is't many set with different codes, not one
+        "CLASH", "CP", "DD3GVL", "DPA", "EURO", "FNMP", "GPX", "GRC", "GUR", "H17", "JR", "MBP", "MGDC", "MLP", "MPRP", "MPS-AKH", "PTC", "S00", "S99", "SUS", "SWS", "UGIN", "UGL", "V10", "V17", "WMCQ", // need to fix
+        "H09", "PD2", "PD3", "UNH", "CM1", "E02", "V11", "M25", "UST", "IMA", "DD2", "EVG", "DDC", "DDE", "DDD", "DDT", "8EB", "9EB", "CHR" // ok
+             // current testing
+    };
 
-    private static final String[] withMythics = {"M10", "M11", "M12", "M13", "M14", "M15", "ORI",
+    private static final String[] symbolsBasicWithMyth = {"M10", "M11", "M12", "M13", "M14", "M15", "ORI",
-        "ANB",
         "DDF", "DDG", "DDH", "DDI", "DDJ", "DDK", "DDL", "DDM", "DDN",
-        "DD3DVD", "DD3GLV", "DD3JVC", "DDO", "DDP", "DDQ", "DDR", "DDS",
+        "DD3DVD", "DD3JVC", "DDO", "DDP", "DDQ", "DDR", "DDS",
         "ALA", "CON", "ARB",
         "ZEN", "WWK", "ROE",
         "SOM", "MBS", "NPH",
@@ -60,60 +90,64 @@ public class GathererSets implements Iterable<DownloadJob> {
         "SOI", "EMN",
         "KLD", "AER",
         "AKH", "HOU",
+        "XLN", "C17",
+        "RIX", "DOM", "M19", // not released
         "E01"
     };
 
-    private static final String[] onlyMythics = {
+    private static final String[] symbolsOnlyMyth = {
-        "DRB", "V09", "V12", "V12", "V13", "V14", "V15", "V16", "EXP"
+        "DRB", "V09", "V12", "V13", "V14", "V15", "V16", "EXP"
     };
-    private static final String[] onlyMythicsAsSpecial = {
+    private static final String[] symbolsOnlySpecial = {
         "MPS"
     };
 
-    private static final HashMap<String, String> symbolsReplacements = new HashMap<>();
+    private static final HashMap<String, String> codeReplacements = new HashMap<>();
 
     static {
-        symbolsReplacements.put("2ED", "2U");
+        codeReplacements.put("2ED", "2U");
-        symbolsReplacements.put("3ED", "3E");
+        codeReplacements.put("3ED", "3E");
-        symbolsReplacements.put("4ED", "4E");
+        codeReplacements.put("4ED", "4E");
-        symbolsReplacements.put("5ED", "5E");
+        codeReplacements.put("5ED", "5E");
-        symbolsReplacements.put("6ED", "6E");
+        codeReplacements.put("6ED", "6E");
-        symbolsReplacements.put("7ED", "7E");
+        codeReplacements.put("7ED", "7E");
-        symbolsReplacements.put("ALL", "AL");
+        codeReplacements.put("ALL", "AL");
-        symbolsReplacements.put("APC", "AP");
+        codeReplacements.put("APC", "AP");
-        symbolsReplacements.put("ARN", "AN");
+        codeReplacements.put("ARN", "AN");
-        symbolsReplacements.put("ATQ", "AQ");
+        codeReplacements.put("ATQ", "AQ");
-        symbolsReplacements.put("CMA", "CM1");
+        codeReplacements.put("CMA", "CM1");
-        symbolsReplacements.put("DD3DVD", "DD3_DVD");
+        codeReplacements.put("DD3DVD", "DD3_DVD");
-        symbolsReplacements.put("DD3EVG", "DD3_EVG");
+        codeReplacements.put("DD3EVG", "DD3_EVG");
-        symbolsReplacements.put("DD3GLV", "DD3_GLV");
+        codeReplacements.put("DD3JVC", "DD3_JVC");
-        symbolsReplacements.put("DD3JVC", "DD3_JVC");
+        codeReplacements.put("DRK", "DK");
-        symbolsReplacements.put("DRK", "DK");
+        codeReplacements.put("EXO", "EX");
-        symbolsReplacements.put("EXO", "EX");
+        codeReplacements.put("FEM", "FE");
-        symbolsReplacements.put("FEM", "FE");
+        codeReplacements.put("HML", "HM");
-        symbolsReplacements.put("HML", "HM");
+        codeReplacements.put("ICE", "IA");
-        symbolsReplacements.put("ICE", "IA");
+        codeReplacements.put("INV", "IN");
-        symbolsReplacements.put("INV", "IN");
+        codeReplacements.put("LEA", "1E");
-        symbolsReplacements.put("LEA", "1E");
+        codeReplacements.put("LEB", "2E");
-        symbolsReplacements.put("LEB", "2E");
+        codeReplacements.put("LEG", "LE");
-        symbolsReplacements.put("LEG", "LE");
+        codeReplacements.put("MPS", "MPS_KLD");
-        symbolsReplacements.put("MPS", "MPS_KLD");
+        codeReplacements.put("MIR", "MI");
-        symbolsReplacements.put("MIR", "MI");
+        codeReplacements.put("MMQ", "MM");
-        symbolsReplacements.put("MMQ", "MM");
+        codeReplacements.put("NEM", "NE");
-        symbolsReplacements.put("NEM", "NE");
+        codeReplacements.put("ODY", "OD");
-        symbolsReplacements.put("ODY", "OD");
+        codeReplacements.put("PCY", "PR");
-        symbolsReplacements.put("PCY", "PR");
+        codeReplacements.put("PLS", "PS");
-        symbolsReplacements.put("PLS", "PS");
+        codeReplacements.put("POR", "PO");
-        symbolsReplacements.put("POR", "PO");
+        codeReplacements.put("P02", "P2");
-        symbolsReplacements.put("PO2", "P2");
+        codeReplacements.put("PTK", "PK");
-        symbolsReplacements.put("PTK", "PK");
+        codeReplacements.put("STH", "ST");
-        symbolsReplacements.put("STH", "ST");
+        codeReplacements.put("TMP", "TE");
-        symbolsReplacements.put("TMP", "TE");
+        codeReplacements.put("UDS", "CG");
-        symbolsReplacements.put("UDS", "CG");
+        codeReplacements.put("ULG", "GU");
-        symbolsReplacements.put("ULG", "GU");
+        codeReplacements.put("USG", "UZ");
-        symbolsReplacements.put("USG", "UZ");
+        codeReplacements.put("VIS", "VI");
-        symbolsReplacements.put("VIS", "VI");
+        codeReplacements.put("WTH", "WL");
-        symbolsReplacements.put("WTH", "WL");
+        codeReplacements.put("8EB", "8ED"); // inner xmage set for 8th edition
+        codeReplacements.put("9EB", "8ED"); // inner xmage set for 9th edition
+        codeReplacements.put("CHR", "CH");
     }
 
     public GathererSets(String path) {
@@ -124,49 +158,155 @@ public class GathererSets implements Iterable<DownloadJob> {
         }
     }
 
+    // checks for wrong card settings and support (easy to control what all good)
+    private static final HashMap<String, CheckResult> setsToDownload = new HashMap<>();
+    private static final HashMap<String, String> codesToIgnoreCheck = new HashMap<>();
+    static {
+        // xMage have inner sets for 8th and 9th Edition for booster workaround (cards from core game do not include in boosters)
+        // see https://mtg.gamepedia.com/8th_Edition/Core_Game
+        // check must ignore that sets
+        codesToIgnoreCheck.put("8EB", "8th Edition Box");
+        codesToIgnoreCheck.put("9EB", "9th Edition Box");
+    }
+
+    private void CheckSearchResult(String searchCode, ExpansionSet foundedExp, boolean canDownloadTask,
+                                   boolean haveCommon, boolean haveUncommon, boolean haveRare, boolean haveMyth){
+        // duplicated in settings
+        CheckResult res = setsToDownload.get(searchCode);
+
+        if (res != null) {
+            logger.error(String.format("Symbols: founded duplicated code: %s", searchCode));
+        } else {
+            res = new CheckResult(searchCode, foundedExp, haveCommon, haveUncommon, haveRare, haveMyth);
+            setsToDownload.put(searchCode, res);
+        }
+
+        // not found
+        if (foundedExp == null) {
+            logger.error(String.format("Symbols: can't find set by code: %s", searchCode));
+            return;
+        }
+
+        // checks for founded sets only
+
+        // to early to download
+        if (!canDownloadTask){
+            Calendar c = Calendar.getInstance();
+            c.setTime(foundedExp.getReleaseDate());
+            c.add(Calendar.DATE, -1 * DAYS_BEFORE_RELEASE_TO_DOWNLOAD);
+            logger.warn(String.format("Symbols: early to download: %s (%s), available after %s",
+                    searchCode, foundedExp.getName(), c.getTime()));
+        }
+    }
+
+    private void AnalyseSearchResult(){
+        // analyze supported sets and show wrong settings
+        Date startedDate = new Date();
+
+        for (ExpansionSet set : Sets.getInstance().values()) {
+
+            // ignore some inner sets
+            if (codesToIgnoreCheck.get(set.getCode()) != null){
+                continue;
+            }
+
+            CheckResult res = setsToDownload.get(set.getCode());
+
+            // 1. not configured at all
+            if (res == null) {
+                logger.warn(String.format("Symbols: set is not configured: %s (%s)", set.getCode(), set.getName()));
+                continue; // can't do other checks
+            }
+
+            // 2. missing rarity icon:
+            // WARNING, need too much time (60+ secs), only for debug mode
+            if (logger.isDebugEnabled()) {
+                if ((set.getCardsByRarity(Rarity.COMMON).size() > 0) && !res.haveCommon) {
+                    logger.error(String.format("Symbols: set have common cards, but don't download icon: %s (%s)", set.getCode(), set.getName()));
+                }
+                if ((set.getCardsByRarity(Rarity.UNCOMMON).size() > 0) && !res.haveUncommon) {
+                    logger.error(String.format("Symbols: set have uncommon cards, but don't download icon: %s (%s)", set.getCode(), set.getName()));
+                }
+                if ((set.getCardsByRarity(Rarity.RARE).size() > 0) && !res.haveRare) {
+                    logger.error(String.format("Symbols: set have rare cards, but don't download icon: %s (%s)", set.getCode(), set.getName()));
+                }
+                if ((set.getCardsByRarity(Rarity.MYTHIC).size() > 0) && !res.haveMyth) {
+                    logger.error(String.format("Symbols: set have mythic cards, but don't download icon: %s (%s)", set.getCode(), set.getName()));
+                }
+            }
+        }
+
+        Date endedDate = new Date();
+        long secs = (endedDate.getTime() - startedDate.getTime()) / 1000;
+        logger.debug(String.format("Symbols: check completed after %d seconds", secs));
+    }
+
     @Override
     public Iterator<DownloadJob> iterator() {
         Calendar c = Calendar.getInstance();
         c.setTime(new Date());
-        c.add(Calendar.DATE, +14); // Try to load the symbols eralies 14 days before release date
+        c.add(Calendar.DATE, DAYS_BEFORE_RELEASE_TO_DOWNLOAD);
         Date compareDate = c.getTime();
         ArrayList<DownloadJob> jobs = new ArrayList<>();
-        for (String symbol : symbols) {
+        boolean canDownload;
+
+        setsToDownload.clear();
+
+        for (String symbol : symbolsBasic) {
             ExpansionSet exp = Sets.findSet(symbol);
+            canDownload = false;
             if (exp != null && exp.getReleaseDate().before(compareDate)) {
+                canDownload = true;
                 jobs.add(generateDownloadJob(symbol, "C", "C"));
                 jobs.add(generateDownloadJob(symbol, "U", "U"));
                 jobs.add(generateDownloadJob(symbol, "R", "R"));
             }
+            CheckSearchResult(symbol, exp, canDownload, true, true, true, false);
         }
-        for (String symbol : withMythics) {
+
+        for (String symbol : symbolsBasicWithMyth) {
             ExpansionSet exp = Sets.findSet(symbol);
+            canDownload = false;
             if (exp != null && exp.getReleaseDate().before(compareDate)) {
+                canDownload = true;
                 jobs.add(generateDownloadJob(symbol, "C", "C"));
                 jobs.add(generateDownloadJob(symbol, "U", "U"));
                 jobs.add(generateDownloadJob(symbol, "R", "R"));
                 jobs.add(generateDownloadJob(symbol, "M", "M"));
             }
+            CheckSearchResult(symbol, exp, canDownload, true, true, true, true);
         }
-        for (String symbol : onlyMythics) {
+
+        for (String symbol : symbolsOnlyMyth) {
             ExpansionSet exp = Sets.findSet(symbol);
+            canDownload = false;
             if (exp != null && exp.getReleaseDate().before(compareDate)) {
+                canDownload = true;
                 jobs.add(generateDownloadJob(symbol, "M", "M"));
             }
+            CheckSearchResult(symbol, exp, canDownload, false, false, false, true);
         }
-        for (String symbol : onlyMythicsAsSpecial) {
+
+        for (String symbol : symbolsOnlySpecial) {
             ExpansionSet exp = Sets.findSet(symbol);
+            canDownload = false;
             if (exp != null && exp.getReleaseDate().before(compareDate)) {
+                canDownload = true;
                 jobs.add(generateDownloadJob(symbol, "M", "S"));
             }
+            CheckSearchResult(symbol, exp, canDownload, false, false, false, true);
         }
+
+        // check wrong settings
+        AnalyseSearchResult();
+
         return jobs.iterator();
     }
 
     private DownloadJob generateDownloadJob(String set, String rarity, String urlRarity) {
         File dst = new File(outDir, set + '-' + rarity + ".jpg");
-        if (symbolsReplacements.containsKey(set)) {
+        if (codeReplacements.containsKey(set)) {
-            set = symbolsReplacements.get(set);
+            set = codeReplacements.get(set);
         }
         String url = "http://gatherer.wizards.com/Handlers/Image.ashx?type=symbol&set=" + set + "&size=small&rarity=" + urlRarity;
         return new DownloadJob(set + '-' + rarity, fromURL(url), toFile(dst));

Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java

@@ -214,10 +214,18 @@ public enum ScryfallImageSource implements CardImageSource {
 
     @Override
     public String generateURL(CardDownloadData card) throws Exception {
-        return "https://api.scryfall.com/cards/" + formatSetName(card.getSet()) + "/"
+
-                + card.getCollectorId()
+        if (card.isTwoFacedCard()) {
-                + (card.isSecondSide() ? "b" : "")
+            // double faced cards do not supporte by API (need direct link for img)
-                + "?format=image";
+            // example: https://img.scryfall.com/cards/large/en/xln/173b.jpg
+            return "https://img.scryfall.com/cards/large/en/" + formatSetName(card.getSet()) + "/"
+                    + card.getCollectorId() + (card.isSecondSide() ? "b" : "a") + ".jpg";
+        }else {
+            // single face cards support by single API call (redirect to img link)
+            // example: https://api.scryfall.com/cards/xln/121?format=image
+            return "https://api.scryfall.com/cards/" + formatSetName(card.getSet()) + "/"
+                    + card.getCollectorId() + "?format=image";
+        }
     }
 
     @Override

Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/WizardCardsImageSource.java

@@ -451,6 +451,7 @@ public enum WizardCardsImageSource implements CardImageSource {
         setsAliases.put("WMCQ", "World Magic Cup Qualifier");
         setsAliases.put("WTH", "Weatherlight");
         setsAliases.put("WWK", "Worldwake");
+        setsAliases.put("XLN", "Ixalan");
         setsAliases.put("ZEN", "Zendikar");
 
         languageAliases = new HashMap<>();

Mage.Server.Plugins/Mage.Game.PennyDreadfulCommanderFreeForAll/target/maven-archiver/pom.properties

@@ -1,5 +0,0 @@
-#Generated by Maven
-#Sun Oct 22 00:34:49 EDT 2017
-version=1.4.26
-groupId=org.mage
-artifactId=mage-game-pennydreadfulcommanderfreeforall