Tests: migrating to mtgjson v5 (#6808);

Mage.Verify/src/main/java/mage/verify/MtgJsonCard.java

@@ -1,23 +0,0 @@
-package mage.verify;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-
-import java.util.List;
-
-import static mage.verify.MtgJsonService.MTGJSON_IGNORE_NEW_PROPERTIES;
-
-@JsonIgnoreProperties(ignoreUnknown = MTGJSON_IGNORE_NEW_PROPERTIES)
-class MtgJsonCard {
-    // docs: https://mtgjson.com/v4/docs.html
-    public List<String> colorIdentity;
-    public List<String> colors;
-    public String layout;
-    public String manaCost;
-    public String number;
-    public String power;
-    public List<String> subtypes;
-    public List<String> supertypes;
-    public String text;
-    public String toughness;
-    public List<String> types;
-}

Mage.Verify/src/main/java/mage/verify/MtgJsonSet.java

@@ -1,16 +0,0 @@
-package mage.verify;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-
-import java.util.List;
-
-import static mage.verify.MtgJsonService.MTGJSON_IGNORE_NEW_PROPERTIES;
-
-@JsonIgnoreProperties(ignoreUnknown = MTGJSON_IGNORE_NEW_PROPERTIES)
-class MtgJsonSet {
-    public List<MtgJsonCard> cards;
-    public String code;
-    public String name;
-    public String releaseDate;
-    public int totalSetSize;
-}

Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonCard.java

@@ -0,0 +1,32 @@
+package mage.verify.mtgjson;
+
+import java.util.List;
+
+public final class MtgJsonCard {
+    // v5 support
+    // https://mtgjson.com/data-models/card-atomic/
+    // contains only used fields, if you need more for tests then just add it here
+
+    public String name;
+
+    public String faceName;
+    public String side;
+
+    public String manaCost;
+    public List<String> colorIdentity;
+    public List<String> colors;
+
+    public List<String> supertypes;
+    public List<String> types;
+    public List<String> subtypes;
+
+    public String text; // rules splits by \n
+
+    public String loyalty;
+    public String power;
+    public String toughness;
+
+    public Integer edhrecRank;
+    public String layout;
+    public List<String> printings; // set codes with that card
+}

Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonService.java

@@ -1,21 +1,15 @@
-package mage.verify;
+package mage.verify.mtgjson;
 
-import com.fasterxml.jackson.core.type.TypeReference;
+import com.google.gson.Gson;
-import com.fasterxml.jackson.databind.ObjectMapper;
 
-import java.io.File;
+import java.io.*;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
 import java.net.URL;
 import java.net.URLConnection;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
 import java.text.Normalizer;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.HashMap;
+import java.util.stream.Collectors;
-import java.util.List;
-import java.util.Map;
 import java.util.zip.ZipInputStream;
 
 public final class MtgJsonService {
@@ -23,25 +17,8 @@ public final class MtgJsonService {
     public static Map<String, String> mtgJsonToXMageCodes = new HashMap<>();
     public static Map<String, String> xMageToMtgJsonCodes = new HashMap<>();
 
-    public static final boolean MTGJSON_IGNORE_NEW_PROPERTIES = true; // set it to false for full mtgjson checks and research (new fields finds or mtgjson updates)
-
-    // TODO: MtgJson now seems to use upper case set codes only.
-    //  Review the ones beginning with a lower case "p" and remove the redundant entries.
     static {
-        mtgJsonToXMageCodes.put("pPRE", "PPRE");
+        //mtgJsonToXMageCodes.put("pPRE", "PPRE");
-        // mtgJsonToXMageCodes.put("pMPR", "MPRP"); // TODO: The set was split into the individual sets
-        mtgJsonToXMageCodes.put("pGRU", "PGRU");
-        mtgJsonToXMageCodes.put("pFNM", "FNMP");
-        mtgJsonToXMageCodes.put("pELP", "PELP");
-        mtgJsonToXMageCodes.put("pARL", "PARL"); // TODO: What about the other Arena League sets (1999-2006)?
-        mtgJsonToXMageCodes.put("pALP", "PALP");
-        mtgJsonToXMageCodes.put("PO2", "P02");
-        mtgJsonToXMageCodes.put("DD3_JVC", "JVC");
-        mtgJsonToXMageCodes.put("DD3_GVL", "DDD");
-        mtgJsonToXMageCodes.put("DD3_EVG", "EVG");
-        mtgJsonToXMageCodes.put("DD3_DVD", "DDC");
-        mtgJsonToXMageCodes.put("NMS", "NEM");
-        mtgJsonToXMageCodes.put("FRF_UGIN", "UGIN");
 
         // revert search
         for (Map.Entry<String, String> entry : mtgJsonToXMageCodes.entrySet()) {
@@ -49,7 +26,118 @@ public final class MtgJsonService {
         }
     }
 
-    private MtgJsonService() {
+    private static Map<String, MtgJsonCard> loadAllCards() throws IOException {
+        AtomicCardsModel json = readFromZip("AtomicCards.json.zip", AtomicCardsModel.class);
+        return json.prepareIndex();
+    }
+
+    private static Map<String, MtgJsonSet> loadAllSets() throws IOException {
+        AllPrintingsModel json = readFromZip("AllPrintings.json.zip", AllPrintingsModel.class);
+        return json.data;
+    }
+
+    private static <T> T readFromZip(String filename, Class<T> clazz) throws IOException {
+        InputStream stream = MtgJsonService.class.getResourceAsStream(filename);
+        if (stream == null) {
+            File file = new File(filename);
+            if (!file.exists()) {
+                String url = "https://mtgjson.com/api/v5/" + filename;
+                System.out.println("Downloading " + url + " to " + file.getAbsolutePath());
+                URLConnection connection = new URL(url).openConnection();
+                connection.setRequestProperty("user-agent", "xmage");
+                InputStream download = connection.getInputStream();
+                Files.copy(download, file.toPath(), StandardCopyOption.REPLACE_EXISTING);
+                System.out.println("Downloading DONE");
+            } else {
+                System.out.println("Founded file " + filename + " from " + file.getAbsolutePath());
+            }
+            stream = new FileInputStream(file);
+        }
+
+        try (ZipInputStream zipInputStream = new ZipInputStream(stream)) {
+            zipInputStream.getNextEntry();
+            return new Gson().fromJson(new InputStreamReader(zipInputStream), clazz);
+        }
+    }
+
+    public static Map<String, MtgJsonSet> sets() {
+        return SetHolder.sets;
+    }
+
+    public static MtgJsonCard card(String name) {
+        return findReference(CardHolder.cards, name);
+    }
+
+    private static <T> T findReference(Map<String, T> reference, String name) {
+        T ref = reference.get(name);
+        if (ref == null) {
+            name = name.replaceFirst("\\bA[Ee]", "Æ");
+            ref = reference.get(name);
+        }
+        if (ref == null) {
+            name = name.replace("'", "\""); // for Kongming, "Sleeping Dragon" & Pang Tong, "Young Phoenix"
+            ref = reference.get(name);
+        }
+
+        return ref;
+    }
+
+    private static <T> void addAliases(Map<String, T> reference) {
+        Map<String, String> aliases = new HashMap<>();
+        for (String name : reference.keySet()) {
+            String unaccented = stripAccents(name);
+            if (!name.equals(unaccented)) {
+                aliases.put(name, unaccented);
+            }
+        }
+        for (Map.Entry<String, String> mapping : aliases.entrySet()) {
+            reference.put(mapping.getValue(), reference.get(mapping.getKey()));
+        }
+    }
+
+    private static String stripAccents(String str) {
+        String decomposed = Normalizer.normalize(str, Normalizer.Form.NFKD);
+        return decomposed.replaceAll("[\\p{InCombiningDiacriticalMarks}]", "");
+    }
+
+    private static final class AtomicCardsModel {
+
+        // list by card names, each name can havem multiple cards (two faces, different cards with same name from un-sets)
+        public HashMap<String, ArrayList<MtgJsonCard>> data;
+
+        private boolean containsSameNames(ArrayList<MtgJsonCard> list) {
+            Set<String> names = list.stream().map(c -> c.name).collect(Collectors.toSet());
+            return names.size() == 1;
+        }
+
+        public HashMap<String, MtgJsonCard> prepareIndex() {
+            HashMap<String, MtgJsonCard> index = new HashMap<>();
+            for (Map.Entry<String, ArrayList<MtgJsonCard>> rec : data.entrySet()) {
+                if (rec.getValue().size() == 1) {
+                    // normal card
+                    index.put(rec.getKey(), rec.getValue().get(0));
+                } else {
+                    if (containsSameNames(rec.getValue())) {
+                        // un-set cards - same name, but different cards (must be ignored)
+                    } else {
+                        // multi-faces cards
+                        MtgJsonCard mainCard = rec.getValue().stream().filter(c -> c.side.equals("a")).findAny().orElse(null);
+                        if (mainCard != null) {
+                            index.put(mainCard.faceName, mainCard);
+                            for (MtgJsonCard card : rec.getValue()) {
+                                if (card == mainCard) continue;
+                                index.put(card.faceName, card);
+                            }
+                        }
+                    }
+                }
+            }
+            return index;
+        }
+    }
+
+    private static final class AllPrintingsModel {
+        public HashMap<String, MtgJsonSet> data;
     }
 
     private static final class CardHolder {
@@ -100,77 +188,4 @@ public final class MtgJsonService {
         }
     }
 
-    private static Map<String, MtgJsonCard> loadAllCards() throws IOException {
-        return readFromZip("AllCards.json.zip", new TypeReference<Map<String, MtgJsonCard>>() {
-        });
-    }
-
-    private static Map<String, MtgJsonSet> loadAllSets() throws IOException {
-        return readFromZip("AllPrintings.json.zip", new TypeReference<Map<String, MtgJsonSet>>() {
-        });
-    }
-
-    private static <T> T readFromZip(String filename, TypeReference<T> ref) throws IOException {
-        InputStream stream = MtgJsonService.class.getResourceAsStream(filename);
-        if (stream == null) {
-            File file = new File(filename);
-            if (!file.exists()) {
-                System.out.println("Downloading " + filename + " to " + file.getAbsolutePath());
-                URLConnection connection = new URL("https://mtgjson.com/files/" + filename).openConnection();
-                connection.setRequestProperty("user-agent", "xmage");
-                InputStream download = connection.getInputStream();
-                Files.copy(download, file.toPath(), StandardCopyOption.REPLACE_EXISTING);
-                System.out.println("Downloading DONE");
-            } else {
-                System.out.println("Using " + filename + " from " + file.getAbsolutePath());
-            }
-            stream = new FileInputStream(file);
-        }
-        try (ZipInputStream zipInputStream = new ZipInputStream(stream)) {
-            zipInputStream.getNextEntry();
-            return new ObjectMapper().readValue(zipInputStream, ref);
-        }
-
-    }
-
-    public static Map<String, MtgJsonSet> sets() {
-        return SetHolder.sets;
-    }
-
-    public static MtgJsonCard card(String name) {
-        return findReference(CardHolder.cards, name);
-    }
-
-    private static <T> T findReference(Map<String, T> reference, String name) {
-        T ref = reference.get(name);
-        if (ref == null) {
-            name = name.replaceFirst("\\bA[Ee]", "Æ");
-            ref = reference.get(name);
-        }
-        if (ref == null) {
-            name = name.replace("'", "\""); // for Kongming, "Sleeping Dragon" & Pang Tong, "Young Phoenix"
-            ref = reference.get(name);
-        }
-
-        return ref;
-    }
-
-    private static <T> void addAliases(Map<String, T> reference) {
-        Map<String, String> aliases = new HashMap<>();
-        for (String name : reference.keySet()) {
-            String unaccented = stripAccents(name);
-            if (!name.equals(unaccented)) {
-                aliases.put(name, unaccented);
-            }
-        }
-        for (Map.Entry<String, String> mapping : aliases.entrySet()) {
-            reference.put(mapping.getValue(), reference.get(mapping.getKey()));
-        }
-    }
-
-    private static String stripAccents(String str) {
-        String decomposed = Normalizer.normalize(str, Normalizer.Form.NFKD);
-        return decomposed.replaceAll("[\\p{InCombiningDiacriticalMarks}]", "");
-    }
-
 }

Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonSet.java

@@ -0,0 +1,15 @@
+package mage.verify.mtgjson;
+
+import java.util.List;
+
+public final class MtgJsonSet {
+    // v5 support
+    // https://mtgjson.com/data-models/card-atomic/
+    // contains only used fields, if you need more for tests then just add it here
+
+    public List<MtgJsonCard> cards;
+    public String code;
+    public String name;
+    public String releaseDate;
+    public int totalSetSize;
+}

Mage.Verify/src/test/java/mage/verify/VerifyCardDataTest.java

@@ -22,6 +22,9 @@ import mage.game.draft.RateCard;
 import mage.game.permanent.token.Token;
 import mage.game.permanent.token.TokenImpl;
 import mage.sets.TherosBeyondDeath;
+import mage.verify.mtgjson.MtgJsonCard;
+import mage.verify.mtgjson.MtgJsonService;
+import mage.verify.mtgjson.MtgJsonSet;
 import mage.watchers.Watcher;
 import org.apache.log4j.Logger;
 import org.junit.Assert;
@@ -425,6 +428,9 @@ public class VerifyCardDataTest {
         int xmageUnofficialCards = 0;
         Collection<ExpansionSet> sets = Sets.getInstance().values();
 
+        Assert.assertFalse("XMage data must contains sets list", sets.isEmpty());
+        Assert.assertFalse("MtgJson data must contains sets list", MtgJsonService.sets().isEmpty());
+
         // official sets
         for (Map.Entry<String, MtgJsonSet> refEntry : MtgJsonService.sets().entrySet()) {
             MtgJsonSet refSet = refEntry.getValue();
@@ -1418,18 +1424,6 @@ public class VerifyCardDataTest {
         }
     }
 
-    private void checkNumbers(Card card, MtgJsonCard ref) {
-        if (skipListHaveName(SKIP_LIST_NUMBER, card.getExpansionSetCode(), card.getName())) {
-            return;
-        }
-
-        String expected = ref.number;
-        String current = card.getCardNumber();
-        if (!eqPT(current, expected)) {
-            warn(card, "card number " + current + " != " + expected);
-        }
-    }
-
     private boolean isBasicLandName(String name) {
 
         String checkName = name;