Tokens rework:

- now token images chosen by tokens database instead availableImageSetCodes (related to #10139);
 - added additional verify checks for tokens database;
 - fixed some tokens;

Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSupportTokens.java

@@ -1422,7 +1422,7 @@ public class ScryfallImageSupportTokens {
 
             // CNS
             put("CNS/Construct", "https://api.scryfall.com/cards/tcns/8/en?format=image");
-            put("CNS/Emblem Dack Fayden", "https://api.scryfall.com/cards/tcns/9/en?format=image");
+            put("CNS/Emblem Dack", "https://api.scryfall.com/cards/tcns/9/en?format=image");
             put("CNS/Demon", "https://api.scryfall.com/cards/tcns/2/en?format=image");
             put("CNS/Elephant", "https://api.scryfall.com/cards/tcns/5/en?format=image");
             put("CNS/Spirit", "https://api.scryfall.com/cards/tcns/1/en?format=image");
@@ -1521,7 +1521,7 @@ public class ScryfallImageSupportTokens {
 
             // EMA
             put("EMA/Carnivore", "https://api.scryfall.com/cards/tema/7/en?format=image");
-            put("EMA/Emblem Dack Fayden", "https://api.scryfall.com/cards/tema/16/en?format=image");
+            put("EMA/Emblem Dack", "https://api.scryfall.com/cards/tema/16/en?format=image");
             put("EMA/Dragon", "https://api.scryfall.com/cards/tema/8/en?format=image");
             put("EMA/Elemental/1", "https://api.scryfall.com/cards/tema/9/en?format=image");
             put("EMA/Elemental/2", "https://api.scryfall.com/cards/tema/14/en?format=image");

Mage.Client/src/main/java/org/mage/plugins/card/images/DownloadPicturesService.java

@@ -538,7 +538,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
             });
 
             // tokens
-            TokenRepository.instance.getAllTokens().forEach(token -> {
+            TokenRepository.instance.getAll().forEach(token -> {
                 CardDownloadData card = new CardDownloadData(
                         token.getName(),
                         token.getSetCode(),
@@ -572,135 +572,6 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
         return Collections.synchronizedList(new ArrayList<>(cardsToDownload));
     }
 
-    public static List<CardDownloadData> getTokenCardUrls() throws RuntimeException {
-        // Must load tokens data in strict mode (throw exception on any error)
-        // Try to put verify checks here instead verify tests
-        String dbSource = "card-pictures-tok.txt";
-        List<CardDownloadData> list = new ArrayList<>();
-        InputStream in = DownloadPicturesService.class.getClassLoader().getResourceAsStream(dbSource);
-        if (in == null) {
-            throw new RuntimeException("Tokens database: can't load resource file " + dbSource);
-        }
-
-        List<String> errorsList = new ArrayList<>();
-        try (InputStreamReader input = new InputStreamReader(in);
-             BufferedReader reader = new BufferedReader(input)) {
-            String line = reader.readLine();
-            while (line != null) {
-                try {
-                    line = line.trim();
-                    if (!line.startsWith("|")) {
-                        continue;
-                    }
-
-                    List<String> params = Arrays.stream(line.split("\\|", -1))
-                            .map(String::trim)
-                            .collect(Collectors.toList());
-                    if (params.size() < 5) {
-                        errorsList.add("Tokens database: wrong params count: " + line);
-                        continue;
-                    }
-                    if (!params.get(1).toLowerCase(Locale.ENGLISH).equals("generate")) {
-                        // TODO: remove "generate" from db
-                        errorsList.add("Tokens database: miss generate param: " + line);
-                        continue;
-                    }
-
-                    // token type (uses if one set contains multiple tokens with same name)
-                    int tokenType = 0;
-                    if (!params.get(4).isEmpty()) {
-                        tokenType = Integer.parseInt(params.get(4));
-                    }
-
-                    // image file name
-                    String imageFileName = "";
-                    if (params.size() > 5 && !params.get(5).isEmpty()) {
-                        imageFileName = params.get(5);
-                    }
-
-                    // token class name (uses for images search for render)
-                    String tokenClassName = "";
-                    if (params.size() > 7 && !params.get(6).isEmpty()) {
-                        tokenClassName = params.get(6);
-                    }
-                    if (tokenClassName.isEmpty()) {
-                        errorsList.add("Tokens database: miss class name: " + line);
-                        continue;
-                    }
-
-                    // object type
-                    String objectType = params.get(2);
-                    String tokenName = params.get(3);
-                    String setCode = "";
-
-                    // type - token
-                    if (objectType.startsWith("TOK:")) {
-                        setCode = objectType.substring("TOK:".length());
-                    }
-
-                    // type - emblem
-                    if (objectType.startsWith("EMBLEM:")) {
-                        setCode = objectType.substring("EMBLEM:".length());
-                        if (!tokenName.startsWith("Emblem ")) {
-                            errorsList.add("Tokens database: emblem's name must start with [Emblem ...] word: " + line);
-                            continue;
-                        }
-                        if (!tokenClassName.endsWith("Emblem")) {
-                            errorsList.add("Tokens database: emblem's class name must ends with [...Emblem] word: " + line);
-                            continue;
-                        }
-                    }
-
-                    // type - plane
-                    if (objectType.startsWith("PLANE:")) {
-                        setCode = objectType.substring("PLANE:".length());
-                        if (!tokenName.startsWith("Plane - ")) {
-                            errorsList.add("Tokens database: plane's name must start with [Plane - ...] word: " + line);
-                            continue;
-                        }
-                        if (!tokenClassName.endsWith("Plane")) {
-                            errorsList.add("Tokens database: plane's class name must ends with [...Plane] word: " + line);
-                            continue;
-                        }
-                    }
-
-                    // type - dungeon
-                    if (objectType.startsWith("DUNGEON:")) {
-                        setCode = objectType.substring("DUNGEON:".length());
-                        if (!tokenClassName.endsWith("Dungeon")) {
-                            errorsList.add("Tokens database: dungeon's class name must ends with [...Dungeon] word: " + line);
-                            continue;
-                        }
-                    }
-
-                    // type - unknown
-                    if (setCode.isEmpty()) {
-                        errorsList.add("Tokens database: unknown line format: " + line);
-                        continue;
-                    }
-
-                    // OK
-                    CardDownloadData card = new CardDownloadData(tokenName, setCode, "0", false, tokenType, true);
-                    card.setTokenClassName(tokenClassName);
-                    card.setFileName(imageFileName);
-                    list.add(card);
-                } finally {
-                    line = reader.readLine();
-                }
-            }
-        } catch (Exception e) {
-            e.printStackTrace();
-            throw new RuntimeException("Tokens database: can't read data, unknown error - " + e.getMessage());
-        }
-
-        if (!errorsList.isEmpty()) {
-            errorsList.forEach(logger::error);
-            throw new RuntimeException(String.format("Tokens database: found %d errors, see logs above for details", errorsList.size()));
-        }
-
-        return list;
-    }
-
     @Override
     public void run() {
         this.cardIndex = 0;

Mage.Tests/src/test/java/org/mage/test/serverside/TokenImagesTest.java

@@ -21,7 +21,7 @@ import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 /**
- * All tests logic: create a tokens list from specific cards and check a used settins (set code, type)
+ * All tests logic: create a tokens list from specific cards and check a used settings (set code, image number)
  *
  * @author JayDi85
  */

Mage.Verify/src/test/java/mage/verify/VerifyCardDataTest.java

@@ -1222,7 +1222,7 @@ public class VerifyCardDataTest {
 
 
         // tok file's data
-        List<TokenInfo> tokFileTokens = TokenRepository.instance.getAllTokens();
+        List<TokenInfo> tokFileTokens = TokenRepository.instance.getAll();
         LinkedHashMap<String, String> tokDataClassesIndex = new LinkedHashMap<>();
         LinkedHashMap<String, String> tokDataNamesIndex = new LinkedHashMap<>();
         LinkedHashMap<String, List<TokenInfo>> tokDataTokensBySetIndex = new LinkedHashMap<>();
@@ -1361,6 +1361,15 @@ public class VerifyCardDataTest {
             }
         });
 
+        // CHECK: token and class names must be same in all sets
+        TokenRepository.instance.getAllByClassName().forEach((className, list) -> {
+            Set<String> names = list.stream().map(TokenInfo::getName).collect(Collectors.toSet());
+            if (names.size() > 1) {
+                errorsList.add("error, card-pictures-tok.txt contains different names for same class: "
+                        + className + " - " + String.join(", ", names));
+            }
+        });
+
         // TODO: all sets must have full tokens data in tok file (token in every set)
         // 1. Download scryfall tokens list: https://api.scryfall.com/cards/search?q=t:token
         // 2. Proccess each token with all prints: read "prints_search_uri" field from token data and go to link like

Mage/src/main/java/mage/cards/repository/RepositoryUtil.java

@@ -26,7 +26,7 @@ public final class RepositoryUtil {
         logger.info("Loading database...");
         ExpansionRepository.instance.getContentVersionConstant();
         CardRepository.instance.getContentVersionConstant();
-        TokenRepository.instance.getAllTokens().size();
+        TokenRepository.instance.getAll().size();
 
         // stats
         int totalCards = CardRepository.instance.findCards(new CardCriteria().nightCard(false)).size()

Mage/src/main/java/mage/cards/repository/TokenInfo.java

@@ -29,6 +29,11 @@ public class TokenInfo {
         this.imageFileName = imageFileName;
     }
 
+    @Override
+    public String toString() {
+        return String.format("%s - %s - %d (%s)", this.setCode, this.name, this.imageNumber, this.classFileName);
+    }
+
     public TokenType getTokenType() {
         return tokenType;
     }

Mage/src/main/java/mage/cards/repository/TokenRepository.java

@@ -25,19 +25,20 @@ public enum TokenRepository {
     }
 
     public void init() {
-        allTokens.clear();
+        if (!allTokens.isEmpty()) {
-        indexByClassName.clear();
+            return;
-        indexByType.clear();
+        }
 
         allTokens = loadAllTokens();
 
         // index
         allTokens.forEach(token -> {
             // by class
-            List<TokenInfo> list = indexByClassName.getOrDefault(token.getClassFileName(), null);
+            String needClass = token.getFullClassFileName();
+            List<TokenInfo> list = indexByClassName.getOrDefault(needClass, null);
             if (list == null) {
                 list = new ArrayList<>();
-                indexByClassName.put(token.getClassFileName(), list);
+                indexByClassName.put(needClass, list);
             }
             list.add(token);
 
@@ -51,18 +52,26 @@ public enum TokenRepository {
         });
     }
 
-    public List<TokenInfo> getAllTokens() {
+    public List<TokenInfo> getAll() {
-        if (allTokens.isEmpty()) {
+        init();
-            init();
-        }
-
         return allTokens;
     }
 
+    public Map<String, List<TokenInfo>> getAllByClassName() {
+        init();
+        return indexByClassName;
+    }
+
     public List<TokenInfo> getByType(TokenType tokenType) {
+        init();
         return indexByType.getOrDefault(tokenType, new ArrayList<>());
     }
 
+    public List<TokenInfo> getByClassName(String fullClassName) {
+        init();
+        return indexByClassName.getOrDefault(fullClassName, new ArrayList<>());
+    }
+
     private static ArrayList<TokenInfo> loadAllTokens() throws RuntimeException {
         // Must load tokens data in strict mode (throw exception on any error)
         // Try to put verify checks here instead verify tests

Mage/src/main/java/mage/game/permanent/token/TokenImpl.java

@@ -9,6 +9,9 @@ import mage.abilities.effects.Effect;
 import mage.abilities.effects.common.AttachEffect;
 import mage.abilities.keyword.EnchantAbility;
 import mage.cards.Card;
+import mage.cards.repository.TokenInfo;
+import mage.cards.repository.TokenRepository;
+import mage.cards.repository.TokenType;
 import mage.constants.Outcome;
 import mage.constants.SubType;
 import mage.constants.Zone;
@@ -24,6 +27,7 @@ import mage.target.Target;
 import mage.util.RandomUtil;
 
 import java.util.*;
+import java.util.stream.Collectors;
 
 /**
  * Each token must have default constructor without params (GUI require for card viewer)
@@ -39,7 +43,7 @@ public abstract class TokenImpl extends MageObjectImpl implements Token {
     private static final int MAX_TOKENS_PER_GAME = 500;
 
     // list of set codes token images are available for
-    protected List<String> availableImageSetCodes = new ArrayList<>();
+    protected List<String> availableImageSetCodes = new ArrayList<>(); // TODO: delete
 
     public enum Type {
 
@@ -140,42 +144,53 @@ public abstract class TokenImpl extends MageObjectImpl implements Token {
         return putOntoBattlefield(amount, game, source, controllerId, tapped, attacking, null);
     }
 
-    public static String generateSetCode(TokenImpl token, Game game, UUID sourceId) {
+    public static TokenInfo generateTokenInfo(TokenImpl token, Game game, UUID sourceId) {
-        // Choose a set code's by priority:
+        // Choose a token image by priority:
-        // - use source's code
+        // - use source's set code
-        // - use parent source's code (too complicated, so ignore it)
+        // - use parent source's set code (too complicated, so ignore it)
         // - use random set code
         // - use default set code
-        // Token type must be set on set's code update
+
+        if (token.getOriginalCardNumber() != null) {
+            // token from a card, so must use card image instead (example: Embalm ability)
+            return new TokenInfo(TokenType.TOKEN, token.getName(), token.getOriginalExpansionSetCode(), 0);
+        }
 
         // source
-        String setCode = null;
+        final String setCode;
         Card sourceCard = game.getCard(sourceId);
         if (sourceCard != null) {
             setCode = sourceCard.getExpansionSetCode();
-        }
+        } else {
-        MageObject sourceObject = game.getObject(sourceId);
+            MageObject sourceObject = game.getObject(sourceId);
-        if (sourceObject instanceof CommandObject) {
+            if (sourceObject instanceof CommandObject) {
-            setCode = ((CommandObject) sourceObject).getExpansionSetCodeForImage();
+                setCode = ((CommandObject) sourceObject).getExpansionSetCodeForImage();
+            } else {
+                setCode = null;
+            }
         }
 
-        // TODO: change to tokens database
+
-        if (token.availableImageSetCodes.contains(setCode)) {
+        // by set code
-            return setCode;
+        List<TokenInfo> possibleInfo = TokenRepository.instance.getByClassName(token.getClass().getName())
+                .stream()
+                .filter(info -> info.getSetCode().equals(setCode))
+                .collect(Collectors.toList());
+
+        // by random set
+        if (possibleInfo.isEmpty()) {
+            possibleInfo = new ArrayList<>(TokenRepository.instance.getByClassName(token.getClass().getName()));
         }
 
-        // random
+        if (possibleInfo.size() > 0) {
-        if (!token.availableImageSetCodes.isEmpty()) {
+            return RandomUtil.randomFromCollection(possibleInfo);
-            return token.availableImageSetCodes.get(RandomUtil.nextInt(token.availableImageSetCodes.size()));
         }
 
-        // default
+        // unknown token
-        // TODO: implement
+        // TODO: download default tokens for xmage's set and use random images from it
-        if (setCode == null) {
+        //  example: TOK.zip/Creature.1.full.jpg
-            setCode = "DEFAULT";
+        //  example: TOK.zip/Creature.2.full.jpg
-        }
+        return new TokenInfo(TokenType.TOKEN, "Unknown", "XMAGE", 0);
-
-        return setCode;
     }
 
     @Override
@@ -241,9 +256,10 @@ public abstract class TokenImpl extends MageObjectImpl implements Token {
             int amount = entry.getValue();
 
             // choose token's set code due source
-            String setCode = TokenImpl.generateSetCode((TokenImpl) token, game, source == null ? null : source.getSourceId());
+            TokenInfo tokenInfo = TokenImpl.generateTokenInfo((TokenImpl) token, game, source == null ? null : source.getSourceId());
-            token.setOriginalExpansionSetCode(setCode);
+            token.setOriginalExpansionSetCode(tokenInfo.getSetCode());
-            token.setExpansionSetCodeForImage(setCode);
+            token.setExpansionSetCodeForImage(tokenInfo.getSetCode());
+            token.setTokenType(tokenInfo.getImageNumber());
 
             List<Permanent> needTokens = new ArrayList<>();
             List<Permanent> allowedTokens = new ArrayList<>();

Mage/src/main/resources/tokens-database.txt

@@ -23,9 +23,9 @@
 |Generate|EMBLEM:C16|Emblem Daretti|||DarettiScrapSavantEmblem|
 |Generate|EMBLEM:CM2|Emblem Daretti|||DarettiScrapSavantEmblem|
 |Generate|EMBLEM:C19|Emblem Nixilis|||ObNixilisReignitedEmblem|
-|Generate|EMBLEM:CNS|Emblem Dack Fayden||Emblem Dack|DackFaydenEmblem|
+|Generate|EMBLEM:CNS|Emblem Dack|||DackFaydenEmblem|
 |Generate|EMBLEM:DTK|Emblem Narset|||NarsetTranscendentEmblem|
-|Generate|EMBLEM:EMA|Emblem Dack Fayden||Emblem Dack|DackFaydenEmblem|
+|Generate|EMBLEM:EMA|Emblem Dack|||DackFaydenEmblem|
 |Generate|EMBLEM:EMN|Emblem Liliana|||LilianaTheLastHopeEmblem|
 |Generate|EMBLEM:EMN|Emblem Tamiyo|||TamiyoFieldResearcherEmblem|
 |Generate|EMBLEM:KLD|Emblem Chandra|||ChandraTorchOfDefianceEmblem|
@@ -163,7 +163,6 @@
 |Generate|TOK:5ED|Citizen|||CitizenToken|
 |Generate|TOK:5ED|Djinn|||DjinnToken|
 |Generate|TOK:5ED|Goblin|||GoblinToken|
-|Generate|TOK:5ED|Insect|||WaspToken|
 |Generate|TOK:5ED|Serf|||SerfToken|
 |Generate|TOK:5ED|Snake|||SerpentGeneratorSnakeToken|
 |Generate|TOK:5ED|Thrull|||BreedingPitBlackInsectToken|
@@ -173,7 +172,6 @@
 |Generate|TOK:6ED|Citizen|||CitizenToken|
 |Generate|TOK:6ED|Djinn|||DjinnToken|
 |Generate|TOK:6ED|Goblin|||GoblinToken|
-|Generate|TOK:6ED|Insect|||WaspToken|
 |Generate|TOK:6ED|Serf|||SerfToken|
 |Generate|TOK:6ED|Snake|||SnakeToken|
 
@@ -468,7 +466,7 @@
 |Generate|TOK:DDD|Beast|2||BeastToken2|
 |Generate|TOK:DDD|Elephant|||ElephantToken|
 |Generate|TOK:DDE|Hornet|||HornetToken|
-|Generate|TOK:DDE|Phyrexian Minion|||MinionToken|
+|Generate|TOK:DDE|Minion|||MinionToken|
 |Generate|TOK:DDE|Saproling|||SaprolingToken|
 |Generate|TOK:DDF|Soldier|||SoldierToken|
 |Generate|TOK:DDG|Goblin|||GoblinToken|
@@ -658,8 +656,6 @@
 |Generate|TOK:KTK|Warrior|1||WarriorToken|
 |Generate|TOK:KTK|Warrior|2||WarriorToken|
 |Generate|TOK:KTK|Zombie|||ZombieToken|
-|Generate|TOK:LEA|Insect|||WaspToken|
-|Generate|TOK:LEB|Insect|||WaspToken|
 |Generate|TOK:LEG|Demon|||MinorDemonToken|
 |Generate|TOK:LEG|Sand Warrior|||HazezonTamarSandWarriorToken|
 |Generate|TOK:LEG|Snake|||SerpentGeneratorSnakeToken|
@@ -802,7 +798,7 @@
 |Generate|TOK:MMA|Worm|||BlackGreenWormToken|
 |Generate|TOK:MMA|Zombie|||ZombieToken|
 |Generate|TOK:MMQ|Ape|||ApeToken|
-|Generate|TOK:MMQ|Insect Butterfly|||ButterflyToken|
+|Generate|TOK:MMQ|Butterfly|||ButterflyToken|
 |Generate|TOK:MMQ|Insect|||InsectToken|
 |Generate|TOK:MMQ|Saproling|||SaprolingToken|
 |Generate|TOK:MMQ|Snake|||SnakeToken|
@@ -862,7 +858,7 @@
 |Generate|TOK:PC2|Beast|||BeastToken|
 |Generate|TOK:PC2|Boar|||BoarToken|
 |Generate|TOK:PC2|Eldrazi Spawn|||EldraziSpawnToken|
-|Generate|TOK:PC2|Germ|||PhyrexianGermToken|
+|Generate|TOK:PC2|Phyrexian Germ|||PhyrexianGermToken|
 |Generate|TOK:PC2|Goblin|||GoblinToken|
 |Generate|TOK:PC2|Hellion|||HellionToken|
 |Generate|TOK:PC2|Insect|||InsectToken|
@@ -887,7 +883,7 @@
 |Generate|TOK:RAV|Knight|||HuntedDragonKnightToken|
 |Generate|TOK:RAV|Saproling|||SaprolingToken|
 |Generate|TOK:RAV|Spirit|||SpiritWhiteToken|
-|Generate|TOK:RAV|Wolf|||VojaToken|
+|Generate|TOK:RAV|Voja|||VojaToken|
 |Generate|TOK:RAV|Faerie|||FaerieToken|
 |Generate|TOK:RIX|Elemental|1||RekindlingPhoenixToken|
 |Generate|TOK:RIX|Elemental|2||RedElementalToken|
@@ -1242,7 +1238,7 @@
 # OonaQueenFaerieRogueToken is FaerieRogueToken with additional blue color, but ZNC contains only one token - so don't use normal token for it
 #|Generate|TOK:ZNC|Faerie Rogue|||OonaQueenFaerieRogueToken|
 # Germ token uses in chest and antology, but scryfall put it here
-#|Generate|TOK:ZNC|Germ|||PhyrexianGermToken|
+#|Generate|TOK:ZNC|Phyrexian Germ|||PhyrexianGermToken|
 #
 |Generate|TOK:ZNC|Goblin Rogue|||GoblinRogueToken|
 |Generate|TOK:ZNC|Kor Ally|||KorAllyToken|