Build the local card database using Scryfall

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+# Recommend matching the black line length (default 88),
+# rather than using the flake8 default of 79:
+max-line-length = 88
+extend-ignore =
+    # See https://github.com/PyCQA/pycodestyle/issues/373
+    E203,

pyproject.toml

@@ -9,6 +9,7 @@ python = "^3.9"
 tornado = "^6.1"
 aiosqlite = "^0.17.0"
 click = "^8.0.1"
+humanize = "^3.10.0"
 httpx = "^0.18.2"
 
 [tool.poetry.dev-dependencies]
@@ -16,6 +17,9 @@ pytest = "^5.2"
 black = "^21.6b0"
 mypy = "^0.910"
 
+[tool.poetry.scripts]
+tutor = 'tutor.__main__:cli'
+
 [build-system]
 requires = ["poetry>=0.12"]
 build-backend = "poetry.masonry.api"

tables.sql

@@ -1,7 +1,43 @@
 CREATE TABLE IF NOT EXISTS `copies` (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    `id` INTEGER PRIMARY KEY AUTOINCREMENT,
-    scryfallId TEXT,
+    `scryfall_id` TEXT,
-    isFoil INTEGER NOT NULL DEFAULT 0,
+    `isFoil` INTEGER NOT NULL DEFAULT 0,
-    language TEXT,
+    `language` TEXT,
-    condition TEXT
+    `condition` TEXT
+);
+
+CREATE TABLE IF NOT EXISTS `cards` (
+    `scryfall_id` TEXT PRIMARY KEY,
+    `name` TEXT NOT NULL,
+    `set_code` TEXT NOT NULL,
+    `collector_number` TEXT NOT NULL,
+    `rarity` TEXT NOT NULL,
+    `color_identity` TEXT NOT NULL,
+    `foil` INTEGER NOT NULL DEFAULT 0,
+    `nonfoil` INTEGER NOT NULL DEFAULT 1,
+    `variation` INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE TABLE IF NOT EXISTS `sets` (
+    `set_code` TEXT PRIMARY KEY,
+    `name` TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS `rarities` (
+    `rarity` TEXT PRIMARY KEY,
+    `rarity_ord` INTEGER NOT NULL
+);
+
+DELETE FROM `rarities`;
+INSERT INTO `rarities` (`rarity`, `rarity_ord`) VALUES
+('common', 1),
+('uncommon', 2),
+('rare', 3),
+('special', 4),
+('mythic', 5),
+('bonus', 6);
+
+CREATE TABLE IF NOT EXISTS `vars` (
+    `key` TEXT PRIMARY KEY,
+    `value` TEXT
 );

tutor/__main__.py

@@ -1,69 +1,17 @@
-import dataclasses
 import json
 import logging
-import typing
 
 import aiosqlite
 import click
+import httpx
+import humanize
 import tornado.ioloop
 import tornado.web
 
 import tutor.csvimport
-
+import tutor.database
-
+import tutor.scryfall
-@dataclasses.dataclass
+import tutor.server
-class Card:
-    uuid: str
-    name: str
-    set_code: str
-
-
-async def search(
-    db: aiosqlite.Connection,
-    name: typing.Optional[str],
-) -> typing.List[Card]:
-    db.row_factory = aiosqlite.Row
-    constraints = []
-    params = {}
-    if name:
-        constraints.append("cards.name LIKE :name")
-        params["name"] = f"%{name}%"
-    query = " ".join(
-        [
-            "SELECT * FROM cards",
-            "WHERE" if constraints else "",
-            " AND ".join(constraints),
-            "LIMIT 10",
-        ]
-    )
-    cursor = await db.execute(query, params)
-    rows = await cursor.fetchall()
-    return [
-        Card(
-            uuid=row["uuid"],
-            name=row["name"],
-            set_code=row["setCode"],
-        )
-        for row in rows
-    ]
-
-
-class SearchHandler(tornado.web.RequestHandler):
-    async def get(self) -> None:
-        async with aiosqlite.connect("/media/correlr/Correl/AllPrintings.sqlite") as db:
-            cards = await search(db, name=self.get_argument("name", None))
-        self.write(
-            json.dumps(
-                [
-                    {
-                        "uuid": card.uuid,
-                        "name": card.name,
-                        "set_code": card.set_code,
-                    }
-                    for card in cards
-                ]
-            )
-        )
 
 
 @click.group()
@@ -91,15 +39,12 @@ def cli(ctx, database, log_level):
 
 
 @cli.command()
+@click.option("--port", type=int, default=8888)
+@click.option("--debug", is_flag=True)
 @click.pass_context
-def server(ctx):
+def server(ctx, port, debug):
-    app = tornado.web.Application(
+    app = tutor.server.make_app({**ctx.obj, "debug": debug})
-        [
+    app.listen(port)
-            (r"/", SearchHandler),
-        ],
-        debug=True,
-    )
-    app.listen(8888)
     tornado.ioloop.IOLoop.current().start()
 
 
@@ -107,10 +52,54 @@ def server(ctx):
 @click.argument("filename", type=click.Path(dir_okay=False))
 @click.pass_context
 def import_cards(ctx, filename):
-    cards = tornado.ioloop.IOLoop.current().run_sync(
+    tornado.ioloop.IOLoop.current().run_sync(
         lambda: tutor.csvimport.load(ctx.obj, filename)
     )
 
 
+@cli.command("update_scryfall")
+@click.option("--filename", type=click.Path(dir_okay=False))
+@click.pass_context
+def update_scryfall(ctx, filename):
+    if filename:
+        with open(filename) as f:
+            cards = json.loads(f.read())
+    else:
+        response = httpx.get("https://api.scryfall.com/bulk-data/oracle_cards")
+        info = response.json()
+
+        buffer = b""
+        with httpx.stream("GET", info["download_uri"]) as response:
+            downloaded = response.num_bytes_downloaded
+            total = int(response.headers["Content-Length"])
+            with click.progressbar(
+                length=total,
+                label=f"Downloading {humanize.naturalsize(total)}"
+                " of card data from Scryfall",
+            ) as bar:
+                for chunk in response.iter_bytes():
+                    buffer += chunk
+                    bar.update(response.num_bytes_downloaded - downloaded)
+                    downloaded = response.num_bytes_downloaded
+        cards = json.loads(buffer)
+
+    async def import_cards():
+        async with aiosqlite.connect(ctx.obj["database"]) as db:
+            with click.progressbar(
+                cards,
+                label=f"Importing {humanize.intcomma(len(cards))} cards",
+            ) as bar:
+                for card_object in bar:
+                    await tutor.database.store_card(
+                        db, tutor.scryfall.to_card(card_object)
+                    )
+                    await tutor.database.store_set(
+                        db, card_object["set"].upper(), card_object["set_name"]
+                    )
+            await db.commit()
+
+    tornado.ioloop.IOLoop.current().run_sync(import_cards)
+
+
 if __name__ == "__main__":
     cli()

tutor/csvimport.py

@@ -3,35 +3,11 @@ import logging
 import typing
 
 import aiosqlite
-import httpx
 
 import tutor.database
 import tutor.models
 
 
-async def find_by_scryfall_id(
-    settings: dict, scryfall_id: str
-) -> typing.List[tutor.models.Card]:
-    async with aiosqlite.connect(settings["database"]) as db:
-        found = await tutor.database.search(db, scryfall_id=row["Scryfall ID"])
-        if not found:
-            # Scryfall IDs could be pointing to a card in another language, which
-            # won't be in our database. Let's get the card info from Scryfall and
-            # search with that.
-            async with httpx.AsyncClient() as client:
-                response = client.get(f"https://api.scryfall.com/card/{scryfall_id}")
-            if response.status_code != httpx.codes.OK:
-                logging.error("Error response from Scryfall: {response}")
-                return []
-            info = response.json()
-            found = await tutor.database.search(
-                db,
-                collector_number=info["collector_number"],
-                set_code=info["set"].upper(),
-            )
-    return found
-
-
 async def load(settings: dict, filename: str) -> typing.List[tutor.models.Card]:
     """Load cards from a CSV file.
 

tutor/database.py

@@ -25,29 +25,27 @@ async def search(
         constraints.append("cards.name LIKE :name")
         params["name"] = name
     if collector_number is not None:
-        constraints.append("cards.number LIKE :number")
+        constraints.append("cards.collector_number LIKE :number")
         params["number"] = collector_number
     if set_code is not None:
-        constraints.append("cards.setCode LIKE :set_code")
+        constraints.append("cards.set_code LIKE :set_code")
-        params["set_code"] = set_code
+        params["set_code"] = set_code.upper()
     if set_name is not None:
         constraints.append("sets.name LIKE :set_name")
         params["set_name"] = set_name
     if foil is not None:
-        constraints.append("cards.hasFoil IS :foil")
+        constraints.append("cards.foil IS :foil")
         params["foil"] = foil
     if alternate_art is not None:
-        constraints.append("cards.isAlternative IS :alternative")
+        constraints.append("cards.variation IS :alternative")
         params["alternative"] = alternate_art
     if scryfall_id is not None:
-        constraints.append("cards.scryfallId LIKE :scryfall_id")
+        constraints.append("cards.scryfall_id LIKE :scryfall_id")
         params["scryfall_id"] = scryfall_id
-    if distinct:
-        constraints.append("(cards.side IS NULL OR cards.side IS 'a')")
     query = " ".join(
         [
             "SELECT cards.* FROM cards",
-            "JOIN sets ON (cards.setCode = sets.code)",
+            "JOIN sets USING (set_code)",
             "WHERE" if constraints else "",
             " AND ".join(constraints),
             f"LIMIT {limit}",
@@ -57,18 +55,53 @@ async def search(
     rows = await cursor.fetchall()
     return [
         tutor.models.Card(
-            scryfall_id=uuid.UUID(row["scryfallId"]),
+            scryfall_id=uuid.UUID(row["scryfall_id"]),
             name=row["name"],
-            set_code=row["setCode"],
+            set_code=row["set_code"],
-            collector_number=row["number"],
+            collector_number=row["collector_number"],
+            rarity=tutor.models.Rarity.from_string(row["rarity"]),
+            color_identity=tutor.models.Color.from_string(row["color_identity"]),
         )
         for row in rows
     ]
 
 
+async def store_card(db: aiosqlite.Connection, card: tutor.models.Card) -> None:
+    await db.execute(
+        "INSERT INTO cards "
+        "(`scryfall_id`, `name`, `set_code`, `collector_number`, `rarity`,"
+        " `color_identity`) "
+        "VALUES (:scryfall_id, :name, :set_code, :collector_number, :rarity,"
+        " :color_identity) "
+        "ON CONFLICT (scryfall_id) DO UPDATE "
+        "SET `name` = :name,"
+        "    `set_code` = :set_code,"
+        "    `collector_number` = :collector_number,"
+        "    `rarity` = :rarity,"
+        "    `color_identity` = :color_identity",
+        {
+            "scryfall_id": str(card.scryfall_id),
+            "name": card.name,
+            "set_code": card.set_code,
+            "collector_number": card.collector_number,
+            "rarity": str(card.rarity),
+            "color_identity": tutor.models.Color.to_string(card.color_identity),
+        },
+    )
+
+
+async def store_set(db: aiosqlite.Connection, set_code: str, name: str) -> None:
+    await db.execute(
+        "INSERT INTO `sets` (`set_code`, `name`) "
+        "VALUES (:set_code, :name) "
+        "ON CONFLICT (`set_code`) DO NOTHING",
+        {"set_code": set_code, "name": name},
+    )
+
+
 async def store_copy(db: aiosqlite.Connection, copy: tutor.models.CardCopy) -> None:
     await db.execute(
-        "INSERT INTO copies (scryfallId, isFoil, condition)"
+        "INSERT INTO copies (scryfall_id, isFoil, condition)"
         "VALUES (:scryfall_id, :foil, :condition)",
         {
             "scryfall_id": str(copy.card.scryfall_id),

tutor/models.py

@@ -1,13 +1,60 @@
 import dataclasses
+import enum
 import typing
 import uuid
 
+
+class Color(enum.IntEnum):
+    White = 1
+    Blue = 2
+    Black = 3
+    Green = 4
+    Red = 5
+
+    def __str__(self) -> str:
+        return dict(zip(Color, "WUBGR")).get(self.value)
+
+    @staticmethod
+    def to_string(colors: typing.List["Color"]) -> str:
+        return "".join(map(str, sorted(colors)))
+
+    @staticmethod
+    def from_string(colors: str) -> typing.List["Color"]:
+        return [
+            color
+            for color in [dict(zip("WUBGR", Color)).get(c) for c in colors.upper()]
+            if color is not None
+        ]
+
+
+class Rarity(enum.IntEnum):
+    Common = 1
+    Uncommon = 2
+    Rare = 3
+    Special = 4
+    Mythic = 5
+    Bonus = 6
+
+    def __str__(self):
+        return self.name.lower()
+
+    @staticmethod
+    def from_string(rarity: str) -> "Rarity":
+        try:
+            return {r.name: r for r in Rarity}[rarity.capitalize()]
+        except KeyError:
+            raise TypeError(f"Unknown rarity '{rarity}'")
+
+
 @dataclasses.dataclass
 class Card:
     scryfall_id: uuid.UUID
     name: str
     set_code: str
     collector_number: str
+    rarity: str
+    color_identity: typing.List[Color]
+
 
 @dataclasses.dataclass
 class CardCopy:

tutor/scryfall.py

@@ -0,0 +1,12 @@
+import tutor.models
+
+
+def to_card(data: dict) -> tutor.models.Card:
+    return tutor.models.Card(
+        scryfall_id=data["id"],
+        name=data["name"],
+        set_code=data["set"].upper(),
+        collector_number=data["collector_number"],
+        rarity=tutor.models.Rarity.from_string(data["rarity"]),
+        color_identity=tutor.models.Color.from_string("".join(data["color_identity"])),
+    )

tutor/server.py

@@ -0,0 +1,42 @@
+import json
+
+import aiosqlite
+import tornado.web
+
+import tutor.database
+import tutor.models
+
+
+class SearchHandler(tornado.web.RequestHandler):
+    async def get(self) -> None:
+        async with aiosqlite.connect(self.application.settings["database"]) as db:
+            cards = await tutor.database.search(
+                db, name=self.get_argument("name", None)
+            )
+        self.set_header("Content-Type", "application/json")
+        self.write(
+            json.dumps(
+                [
+                    {
+                        "scryfall_id": str(card.scryfall_id),
+                        "name": card.name,
+                        "set_code": card.set_code,
+                        "collector_number": card.collector_number,
+                        "rarity": card.rarity,
+                        "color_identity": tutor.models.Color.to_string(
+                            card.color_identity
+                        ),
+                    }
+                    for card in cards
+                ]
+            )
+        )
+
+
+def make_app(settings) -> tornado.web.Application:
+    return tornado.web.Application(
+        [
+            (r"/", SearchHandler),
+        ],
+        **settings,
+    )