From ca254c65622c5f134a9f0518ea483e1dd1e43bae Mon Sep 17 00:00:00 2001 From: Oleg Agafonov Date: Sat, 15 Aug 2020 14:21:38 +0400 Subject: [PATCH] Tests: migrating to mtgjson v5 (#6808); --- .../main/java/mage/verify/MtgJsonCard.java | 23 -- .../src/main/java/mage/verify/MtgJsonSet.java | 16 -- .../java/mage/verify/mtgjson/MtgJsonCard.java | 32 +++ .../verify/{ => mtgjson}/MtgJsonService.java | 221 ++++++++++-------- .../java/mage/verify/mtgjson/MtgJsonSet.java | 15 ++ .../java/mage/verify/VerifyCardDataTest.java | 18 +- 6 files changed, 171 insertions(+), 154 deletions(-) delete mode 100644 Mage.Verify/src/main/java/mage/verify/MtgJsonCard.java delete mode 100644 Mage.Verify/src/main/java/mage/verify/MtgJsonSet.java create mode 100644 Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonCard.java rename Mage.Verify/src/main/java/mage/verify/{ => mtgjson}/MtgJsonService.java (62%) create mode 100644 Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonSet.java diff --git a/Mage.Verify/src/main/java/mage/verify/MtgJsonCard.java b/Mage.Verify/src/main/java/mage/verify/MtgJsonCard.java deleted file mode 100644 index 6e990337d2..0000000000 --- a/Mage.Verify/src/main/java/mage/verify/MtgJsonCard.java +++ /dev/null @@ -1,23 +0,0 @@ -package mage.verify; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; - -import java.util.List; - -import static mage.verify.MtgJsonService.MTGJSON_IGNORE_NEW_PROPERTIES; - -@JsonIgnoreProperties(ignoreUnknown = MTGJSON_IGNORE_NEW_PROPERTIES) -class MtgJsonCard { - // docs: https://mtgjson.com/v4/docs.html - public List colorIdentity; - public List colors; - public String layout; - public String manaCost; - public String number; - public String power; - public List subtypes; - public List supertypes; - public String text; - public String toughness; - public List types; -} diff --git a/Mage.Verify/src/main/java/mage/verify/MtgJsonSet.java b/Mage.Verify/src/main/java/mage/verify/MtgJsonSet.java deleted file mode 100644 index 38a5a68978..0000000000 --- a/Mage.Verify/src/main/java/mage/verify/MtgJsonSet.java +++ /dev/null @@ -1,16 +0,0 @@ -package mage.verify; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; - -import java.util.List; - -import static mage.verify.MtgJsonService.MTGJSON_IGNORE_NEW_PROPERTIES; - -@JsonIgnoreProperties(ignoreUnknown = MTGJSON_IGNORE_NEW_PROPERTIES) -class MtgJsonSet { - public List cards; - public String code; - public String name; - public String releaseDate; - public int totalSetSize; -} diff --git a/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonCard.java b/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonCard.java new file mode 100644 index 0000000000..49f7d0756f --- /dev/null +++ b/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonCard.java @@ -0,0 +1,32 @@ +package mage.verify.mtgjson; + +import java.util.List; + +public final class MtgJsonCard { + // v5 support + // https://mtgjson.com/data-models/card-atomic/ + // contains only used fields, if you need more for tests then just add it here + + public String name; + + public String faceName; + public String side; + + public String manaCost; + public List colorIdentity; + public List colors; + + public List supertypes; + public List types; + public List subtypes; + + public String text; // rules splits by \n + + public String loyalty; + public String power; + public String toughness; + + public Integer edhrecRank; + public String layout; + public List printings; // set codes with that card +} diff --git a/Mage.Verify/src/main/java/mage/verify/MtgJsonService.java b/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonService.java similarity index 62% rename from Mage.Verify/src/main/java/mage/verify/MtgJsonService.java rename to Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonService.java index fc2a391929..5cd6438498 100644 --- a/Mage.Verify/src/main/java/mage/verify/MtgJsonService.java +++ b/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonService.java @@ -1,21 +1,15 @@ -package mage.verify; +package mage.verify.mtgjson; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.net.URL; import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.StandardCopyOption; import java.text.Normalizer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; import java.util.zip.ZipInputStream; public final class MtgJsonService { @@ -23,25 +17,8 @@ public final class MtgJsonService { public static Map mtgJsonToXMageCodes = new HashMap<>(); public static Map xMageToMtgJsonCodes = new HashMap<>(); - public static final boolean MTGJSON_IGNORE_NEW_PROPERTIES = true; // set it to false for full mtgjson checks and research (new fields finds or mtgjson updates) - - // TODO: MtgJson now seems to use upper case set codes only. - // Review the ones beginning with a lower case "p" and remove the redundant entries. static { - mtgJsonToXMageCodes.put("pPRE", "PPRE"); - // mtgJsonToXMageCodes.put("pMPR", "MPRP"); // TODO: The set was split into the individual sets - mtgJsonToXMageCodes.put("pGRU", "PGRU"); - mtgJsonToXMageCodes.put("pFNM", "FNMP"); - mtgJsonToXMageCodes.put("pELP", "PELP"); - mtgJsonToXMageCodes.put("pARL", "PARL"); // TODO: What about the other Arena League sets (1999-2006)? - mtgJsonToXMageCodes.put("pALP", "PALP"); - mtgJsonToXMageCodes.put("PO2", "P02"); - mtgJsonToXMageCodes.put("DD3_JVC", "JVC"); - mtgJsonToXMageCodes.put("DD3_GVL", "DDD"); - mtgJsonToXMageCodes.put("DD3_EVG", "EVG"); - mtgJsonToXMageCodes.put("DD3_DVD", "DDC"); - mtgJsonToXMageCodes.put("NMS", "NEM"); - mtgJsonToXMageCodes.put("FRF_UGIN", "UGIN"); + //mtgJsonToXMageCodes.put("pPRE", "PPRE"); // revert search for (Map.Entry entry : mtgJsonToXMageCodes.entrySet()) { @@ -49,7 +26,118 @@ public final class MtgJsonService { } } - private MtgJsonService() { + private static Map loadAllCards() throws IOException { + AtomicCardsModel json = readFromZip("AtomicCards.json.zip", AtomicCardsModel.class); + return json.prepareIndex(); + } + + private static Map loadAllSets() throws IOException { + AllPrintingsModel json = readFromZip("AllPrintings.json.zip", AllPrintingsModel.class); + return json.data; + } + + private static T readFromZip(String filename, Class clazz) throws IOException { + InputStream stream = MtgJsonService.class.getResourceAsStream(filename); + if (stream == null) { + File file = new File(filename); + if (!file.exists()) { + String url = "https://mtgjson.com/api/v5/" + filename; + System.out.println("Downloading " + url + " to " + file.getAbsolutePath()); + URLConnection connection = new URL(url).openConnection(); + connection.setRequestProperty("user-agent", "xmage"); + InputStream download = connection.getInputStream(); + Files.copy(download, file.toPath(), StandardCopyOption.REPLACE_EXISTING); + System.out.println("Downloading DONE"); + } else { + System.out.println("Founded file " + filename + " from " + file.getAbsolutePath()); + } + stream = new FileInputStream(file); + } + + try (ZipInputStream zipInputStream = new ZipInputStream(stream)) { + zipInputStream.getNextEntry(); + return new Gson().fromJson(new InputStreamReader(zipInputStream), clazz); + } + } + + public static Map sets() { + return SetHolder.sets; + } + + public static MtgJsonCard card(String name) { + return findReference(CardHolder.cards, name); + } + + private static T findReference(Map reference, String name) { + T ref = reference.get(name); + if (ref == null) { + name = name.replaceFirst("\\bA[Ee]", "Æ"); + ref = reference.get(name); + } + if (ref == null) { + name = name.replace("'", "\""); // for Kongming, "Sleeping Dragon" & Pang Tong, "Young Phoenix" + ref = reference.get(name); + } + + return ref; + } + + private static void addAliases(Map reference) { + Map aliases = new HashMap<>(); + for (String name : reference.keySet()) { + String unaccented = stripAccents(name); + if (!name.equals(unaccented)) { + aliases.put(name, unaccented); + } + } + for (Map.Entry mapping : aliases.entrySet()) { + reference.put(mapping.getValue(), reference.get(mapping.getKey())); + } + } + + private static String stripAccents(String str) { + String decomposed = Normalizer.normalize(str, Normalizer.Form.NFKD); + return decomposed.replaceAll("[\\p{InCombiningDiacriticalMarks}]", ""); + } + + private static final class AtomicCardsModel { + + // list by card names, each name can havem multiple cards (two faces, different cards with same name from un-sets) + public HashMap> data; + + private boolean containsSameNames(ArrayList list) { + Set names = list.stream().map(c -> c.name).collect(Collectors.toSet()); + return names.size() == 1; + } + + public HashMap prepareIndex() { + HashMap index = new HashMap<>(); + for (Map.Entry> rec : data.entrySet()) { + if (rec.getValue().size() == 1) { + // normal card + index.put(rec.getKey(), rec.getValue().get(0)); + } else { + if (containsSameNames(rec.getValue())) { + // un-set cards - same name, but different cards (must be ignored) + } else { + // multi-faces cards + MtgJsonCard mainCard = rec.getValue().stream().filter(c -> c.side.equals("a")).findAny().orElse(null); + if (mainCard != null) { + index.put(mainCard.faceName, mainCard); + for (MtgJsonCard card : rec.getValue()) { + if (card == mainCard) continue; + index.put(card.faceName, card); + } + } + } + } + } + return index; + } + } + + private static final class AllPrintingsModel { + public HashMap data; } private static final class CardHolder { @@ -100,77 +188,4 @@ public final class MtgJsonService { } } - private static Map loadAllCards() throws IOException { - return readFromZip("AllCards.json.zip", new TypeReference>() { - }); - } - - private static Map loadAllSets() throws IOException { - return readFromZip("AllPrintings.json.zip", new TypeReference>() { - }); - } - - private static T readFromZip(String filename, TypeReference ref) throws IOException { - InputStream stream = MtgJsonService.class.getResourceAsStream(filename); - if (stream == null) { - File file = new File(filename); - if (!file.exists()) { - System.out.println("Downloading " + filename + " to " + file.getAbsolutePath()); - URLConnection connection = new URL("https://mtgjson.com/files/" + filename).openConnection(); - connection.setRequestProperty("user-agent", "xmage"); - InputStream download = connection.getInputStream(); - Files.copy(download, file.toPath(), StandardCopyOption.REPLACE_EXISTING); - System.out.println("Downloading DONE"); - } else { - System.out.println("Using " + filename + " from " + file.getAbsolutePath()); - } - stream = new FileInputStream(file); - } - try (ZipInputStream zipInputStream = new ZipInputStream(stream)) { - zipInputStream.getNextEntry(); - return new ObjectMapper().readValue(zipInputStream, ref); - } - - } - - public static Map sets() { - return SetHolder.sets; - } - - public static MtgJsonCard card(String name) { - return findReference(CardHolder.cards, name); - } - - private static T findReference(Map reference, String name) { - T ref = reference.get(name); - if (ref == null) { - name = name.replaceFirst("\\bA[Ee]", "Æ"); - ref = reference.get(name); - } - if (ref == null) { - name = name.replace("'", "\""); // for Kongming, "Sleeping Dragon" & Pang Tong, "Young Phoenix" - ref = reference.get(name); - } - - return ref; - } - - private static void addAliases(Map reference) { - Map aliases = new HashMap<>(); - for (String name : reference.keySet()) { - String unaccented = stripAccents(name); - if (!name.equals(unaccented)) { - aliases.put(name, unaccented); - } - } - for (Map.Entry mapping : aliases.entrySet()) { - reference.put(mapping.getValue(), reference.get(mapping.getKey())); - } - } - - private static String stripAccents(String str) { - String decomposed = Normalizer.normalize(str, Normalizer.Form.NFKD); - return decomposed.replaceAll("[\\p{InCombiningDiacriticalMarks}]", ""); - } - } diff --git a/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonSet.java b/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonSet.java new file mode 100644 index 0000000000..e15acdf206 --- /dev/null +++ b/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonSet.java @@ -0,0 +1,15 @@ +package mage.verify.mtgjson; + +import java.util.List; + +public final class MtgJsonSet { + // v5 support + // https://mtgjson.com/data-models/card-atomic/ + // contains only used fields, if you need more for tests then just add it here + + public List cards; + public String code; + public String name; + public String releaseDate; + public int totalSetSize; +} diff --git a/Mage.Verify/src/test/java/mage/verify/VerifyCardDataTest.java b/Mage.Verify/src/test/java/mage/verify/VerifyCardDataTest.java index 208625951e..e1e8ea430f 100644 --- a/Mage.Verify/src/test/java/mage/verify/VerifyCardDataTest.java +++ b/Mage.Verify/src/test/java/mage/verify/VerifyCardDataTest.java @@ -22,6 +22,9 @@ import mage.game.draft.RateCard; import mage.game.permanent.token.Token; import mage.game.permanent.token.TokenImpl; import mage.sets.TherosBeyondDeath; +import mage.verify.mtgjson.MtgJsonCard; +import mage.verify.mtgjson.MtgJsonService; +import mage.verify.mtgjson.MtgJsonSet; import mage.watchers.Watcher; import org.apache.log4j.Logger; import org.junit.Assert; @@ -425,6 +428,9 @@ public class VerifyCardDataTest { int xmageUnofficialCards = 0; Collection sets = Sets.getInstance().values(); + Assert.assertFalse("XMage data must contains sets list", sets.isEmpty()); + Assert.assertFalse("MtgJson data must contains sets list", MtgJsonService.sets().isEmpty()); + // official sets for (Map.Entry refEntry : MtgJsonService.sets().entrySet()) { MtgJsonSet refSet = refEntry.getValue(); @@ -1418,18 +1424,6 @@ public class VerifyCardDataTest { } } - private void checkNumbers(Card card, MtgJsonCard ref) { - if (skipListHaveName(SKIP_LIST_NUMBER, card.getExpansionSetCode(), card.getName())) { - return; - } - - String expected = ref.number; - String current = card.getCardNumber(); - if (!eqPT(current, expected)) { - warn(card, "card number " + current + " != " + expected); - } - } - private boolean isBasicLandName(String name) { String checkName = name;