diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..f338d00 --- /dev/null +++ b/.flake8 @@ -0,0 +1,7 @@ +[flake8] +# Recommend matching the black line length (default 88), +# rather than using the flake8 default of 79: +max-line-length = 88 +extend-ignore = + # See https://github.com/PyCQA/pycodestyle/issues/373 + E203, diff --git a/pyproject.toml b/pyproject.toml index cd6b196..4ba5780 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ python = "^3.9" tornado = "^6.1" aiosqlite = "^0.17.0" click = "^8.0.1" +humanize = "^3.10.0" httpx = "^0.18.2" [tool.poetry.dev-dependencies] @@ -16,6 +17,9 @@ pytest = "^5.2" black = "^21.6b0" mypy = "^0.910" +[tool.poetry.scripts] +tutor = 'tutor.__main__:cli' + [build-system] requires = ["poetry>=0.12"] build-backend = "poetry.masonry.api" diff --git a/tables.sql b/tables.sql index 0c706ea..3be135f 100644 --- a/tables.sql +++ b/tables.sql @@ -1,7 +1,43 @@ CREATE TABLE IF NOT EXISTS `copies` ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - scryfallId TEXT, - isFoil INTEGER NOT NULL DEFAULT 0, - language TEXT, - condition TEXT + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `scryfall_id` TEXT, + `isFoil` INTEGER NOT NULL DEFAULT 0, + `language` TEXT, + `condition` TEXT +); + +CREATE TABLE IF NOT EXISTS `cards` ( + `scryfall_id` TEXT PRIMARY KEY, + `name` TEXT NOT NULL, + `set_code` TEXT NOT NULL, + `collector_number` TEXT NOT NULL, + `rarity` TEXT NOT NULL, + `color_identity` TEXT NOT NULL, + `foil` INTEGER NOT NULL DEFAULT 0, + `nonfoil` INTEGER NOT NULL DEFAULT 1, + `variation` INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE IF NOT EXISTS `sets` ( + `set_code` TEXT PRIMARY KEY, + `name` TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS `rarities` ( + `rarity` TEXT PRIMARY KEY, + `rarity_ord` INTEGER NOT NULL +); + +DELETE FROM `rarities`; +INSERT INTO `rarities` (`rarity`, `rarity_ord`) VALUES +('common', 1), +('uncommon', 2), +('rare', 3), +('special', 4), +('mythic', 5), +('bonus', 6); + +CREATE TABLE IF NOT EXISTS `vars` ( + `key` TEXT PRIMARY KEY, + `value` TEXT ); diff --git a/tutor/__main__.py b/tutor/__main__.py index 64301ed..ffa7039 100644 --- a/tutor/__main__.py +++ b/tutor/__main__.py @@ -1,69 +1,17 @@ -import dataclasses import json import logging -import typing import aiosqlite import click +import httpx +import humanize import tornado.ioloop import tornado.web import tutor.csvimport - - -@dataclasses.dataclass -class Card: - uuid: str - name: str - set_code: str - - -async def search( - db: aiosqlite.Connection, - name: typing.Optional[str], -) -> typing.List[Card]: - db.row_factory = aiosqlite.Row - constraints = [] - params = {} - if name: - constraints.append("cards.name LIKE :name") - params["name"] = f"%{name}%" - query = " ".join( - [ - "SELECT * FROM cards", - "WHERE" if constraints else "", - " AND ".join(constraints), - "LIMIT 10", - ] - ) - cursor = await db.execute(query, params) - rows = await cursor.fetchall() - return [ - Card( - uuid=row["uuid"], - name=row["name"], - set_code=row["setCode"], - ) - for row in rows - ] - - -class SearchHandler(tornado.web.RequestHandler): - async def get(self) -> None: - async with aiosqlite.connect("/media/correlr/Correl/AllPrintings.sqlite") as db: - cards = await search(db, name=self.get_argument("name", None)) - self.write( - json.dumps( - [ - { - "uuid": card.uuid, - "name": card.name, - "set_code": card.set_code, - } - for card in cards - ] - ) - ) +import tutor.database +import tutor.scryfall +import tutor.server @click.group() @@ -91,15 +39,12 @@ def cli(ctx, database, log_level): @cli.command() +@click.option("--port", type=int, default=8888) +@click.option("--debug", is_flag=True) @click.pass_context -def server(ctx): - app = tornado.web.Application( - [ - (r"/", SearchHandler), - ], - debug=True, - ) - app.listen(8888) +def server(ctx, port, debug): + app = tutor.server.make_app({**ctx.obj, "debug": debug}) + app.listen(port) tornado.ioloop.IOLoop.current().start() @@ -107,10 +52,54 @@ def server(ctx): @click.argument("filename", type=click.Path(dir_okay=False)) @click.pass_context def import_cards(ctx, filename): - cards = tornado.ioloop.IOLoop.current().run_sync( + tornado.ioloop.IOLoop.current().run_sync( lambda: tutor.csvimport.load(ctx.obj, filename) ) +@cli.command("update_scryfall") +@click.option("--filename", type=click.Path(dir_okay=False)) +@click.pass_context +def update_scryfall(ctx, filename): + if filename: + with open(filename) as f: + cards = json.loads(f.read()) + else: + response = httpx.get("https://api.scryfall.com/bulk-data/oracle_cards") + info = response.json() + + buffer = b"" + with httpx.stream("GET", info["download_uri"]) as response: + downloaded = response.num_bytes_downloaded + total = int(response.headers["Content-Length"]) + with click.progressbar( + length=total, + label=f"Downloading {humanize.naturalsize(total)}" + " of card data from Scryfall", + ) as bar: + for chunk in response.iter_bytes(): + buffer += chunk + bar.update(response.num_bytes_downloaded - downloaded) + downloaded = response.num_bytes_downloaded + cards = json.loads(buffer) + + async def import_cards(): + async with aiosqlite.connect(ctx.obj["database"]) as db: + with click.progressbar( + cards, + label=f"Importing {humanize.intcomma(len(cards))} cards", + ) as bar: + for card_object in bar: + await tutor.database.store_card( + db, tutor.scryfall.to_card(card_object) + ) + await tutor.database.store_set( + db, card_object["set"].upper(), card_object["set_name"] + ) + await db.commit() + + tornado.ioloop.IOLoop.current().run_sync(import_cards) + + if __name__ == "__main__": cli() diff --git a/tutor/csvimport.py b/tutor/csvimport.py index ab40d3f..85180a9 100644 --- a/tutor/csvimport.py +++ b/tutor/csvimport.py @@ -3,35 +3,11 @@ import logging import typing import aiosqlite -import httpx import tutor.database import tutor.models -async def find_by_scryfall_id( - settings: dict, scryfall_id: str -) -> typing.List[tutor.models.Card]: - async with aiosqlite.connect(settings["database"]) as db: - found = await tutor.database.search(db, scryfall_id=row["Scryfall ID"]) - if not found: - # Scryfall IDs could be pointing to a card in another language, which - # won't be in our database. Let's get the card info from Scryfall and - # search with that. - async with httpx.AsyncClient() as client: - response = client.get(f"https://api.scryfall.com/card/{scryfall_id}") - if response.status_code != httpx.codes.OK: - logging.error("Error response from Scryfall: {response}") - return [] - info = response.json() - found = await tutor.database.search( - db, - collector_number=info["collector_number"], - set_code=info["set"].upper(), - ) - return found - - async def load(settings: dict, filename: str) -> typing.List[tutor.models.Card]: """Load cards from a CSV file. diff --git a/tutor/database.py b/tutor/database.py index e308a02..9c5a3a8 100644 --- a/tutor/database.py +++ b/tutor/database.py @@ -25,29 +25,27 @@ async def search( constraints.append("cards.name LIKE :name") params["name"] = name if collector_number is not None: - constraints.append("cards.number LIKE :number") + constraints.append("cards.collector_number LIKE :number") params["number"] = collector_number if set_code is not None: - constraints.append("cards.setCode LIKE :set_code") - params["set_code"] = set_code + constraints.append("cards.set_code LIKE :set_code") + params["set_code"] = set_code.upper() if set_name is not None: constraints.append("sets.name LIKE :set_name") params["set_name"] = set_name if foil is not None: - constraints.append("cards.hasFoil IS :foil") + constraints.append("cards.foil IS :foil") params["foil"] = foil if alternate_art is not None: - constraints.append("cards.isAlternative IS :alternative") + constraints.append("cards.variation IS :alternative") params["alternative"] = alternate_art if scryfall_id is not None: - constraints.append("cards.scryfallId LIKE :scryfall_id") + constraints.append("cards.scryfall_id LIKE :scryfall_id") params["scryfall_id"] = scryfall_id - if distinct: - constraints.append("(cards.side IS NULL OR cards.side IS 'a')") query = " ".join( [ "SELECT cards.* FROM cards", - "JOIN sets ON (cards.setCode = sets.code)", + "JOIN sets USING (set_code)", "WHERE" if constraints else "", " AND ".join(constraints), f"LIMIT {limit}", @@ -57,18 +55,53 @@ async def search( rows = await cursor.fetchall() return [ tutor.models.Card( - scryfall_id=uuid.UUID(row["scryfallId"]), + scryfall_id=uuid.UUID(row["scryfall_id"]), name=row["name"], - set_code=row["setCode"], - collector_number=row["number"], + set_code=row["set_code"], + collector_number=row["collector_number"], + rarity=tutor.models.Rarity.from_string(row["rarity"]), + color_identity=tutor.models.Color.from_string(row["color_identity"]), ) for row in rows ] +async def store_card(db: aiosqlite.Connection, card: tutor.models.Card) -> None: + await db.execute( + "INSERT INTO cards " + "(`scryfall_id`, `name`, `set_code`, `collector_number`, `rarity`," + " `color_identity`) " + "VALUES (:scryfall_id, :name, :set_code, :collector_number, :rarity," + " :color_identity) " + "ON CONFLICT (scryfall_id) DO UPDATE " + "SET `name` = :name," + " `set_code` = :set_code," + " `collector_number` = :collector_number," + " `rarity` = :rarity," + " `color_identity` = :color_identity", + { + "scryfall_id": str(card.scryfall_id), + "name": card.name, + "set_code": card.set_code, + "collector_number": card.collector_number, + "rarity": str(card.rarity), + "color_identity": tutor.models.Color.to_string(card.color_identity), + }, + ) + + +async def store_set(db: aiosqlite.Connection, set_code: str, name: str) -> None: + await db.execute( + "INSERT INTO `sets` (`set_code`, `name`) " + "VALUES (:set_code, :name) " + "ON CONFLICT (`set_code`) DO NOTHING", + {"set_code": set_code, "name": name}, + ) + + async def store_copy(db: aiosqlite.Connection, copy: tutor.models.CardCopy) -> None: await db.execute( - "INSERT INTO copies (scryfallId, isFoil, condition)" + "INSERT INTO copies (scryfall_id, isFoil, condition)" "VALUES (:scryfall_id, :foil, :condition)", { "scryfall_id": str(copy.card.scryfall_id), diff --git a/tutor/models.py b/tutor/models.py index 7b42e4f..e6ea70f 100644 --- a/tutor/models.py +++ b/tutor/models.py @@ -1,13 +1,60 @@ import dataclasses +import enum import typing import uuid + +class Color(enum.IntEnum): + White = 1 + Blue = 2 + Black = 3 + Green = 4 + Red = 5 + + def __str__(self) -> str: + return dict(zip(Color, "WUBGR")).get(self.value) + + @staticmethod + def to_string(colors: typing.List["Color"]) -> str: + return "".join(map(str, sorted(colors))) + + @staticmethod + def from_string(colors: str) -> typing.List["Color"]: + return [ + color + for color in [dict(zip("WUBGR", Color)).get(c) for c in colors.upper()] + if color is not None + ] + + +class Rarity(enum.IntEnum): + Common = 1 + Uncommon = 2 + Rare = 3 + Special = 4 + Mythic = 5 + Bonus = 6 + + def __str__(self): + return self.name.lower() + + @staticmethod + def from_string(rarity: str) -> "Rarity": + try: + return {r.name: r for r in Rarity}[rarity.capitalize()] + except KeyError: + raise TypeError(f"Unknown rarity '{rarity}'") + + @dataclasses.dataclass class Card: scryfall_id: uuid.UUID name: str set_code: str collector_number: str + rarity: str + color_identity: typing.List[Color] + @dataclasses.dataclass class CardCopy: diff --git a/tutor/scryfall.py b/tutor/scryfall.py new file mode 100644 index 0000000..e05f22b --- /dev/null +++ b/tutor/scryfall.py @@ -0,0 +1,12 @@ +import tutor.models + + +def to_card(data: dict) -> tutor.models.Card: + return tutor.models.Card( + scryfall_id=data["id"], + name=data["name"], + set_code=data["set"].upper(), + collector_number=data["collector_number"], + rarity=tutor.models.Rarity.from_string(data["rarity"]), + color_identity=tutor.models.Color.from_string("".join(data["color_identity"])), + ) diff --git a/tutor/server.py b/tutor/server.py new file mode 100644 index 0000000..e44f062 --- /dev/null +++ b/tutor/server.py @@ -0,0 +1,42 @@ +import json + +import aiosqlite +import tornado.web + +import tutor.database +import tutor.models + + +class SearchHandler(tornado.web.RequestHandler): + async def get(self) -> None: + async with aiosqlite.connect(self.application.settings["database"]) as db: + cards = await tutor.database.search( + db, name=self.get_argument("name", None) + ) + self.set_header("Content-Type", "application/json") + self.write( + json.dumps( + [ + { + "scryfall_id": str(card.scryfall_id), + "name": card.name, + "set_code": card.set_code, + "collector_number": card.collector_number, + "rarity": card.rarity, + "color_identity": tutor.models.Color.to_string( + card.color_identity + ), + } + for card in cards + ] + ) + ) + + +def make_app(settings) -> tornado.web.Application: + return tornado.web.Application( + [ + (r"/", SearchHandler), + ], + **settings, + )