Build the local card database using Scryfall
This commit is contained in:
parent
996adbf376
commit
abd73e0a89
|
|
@ -0,0 +1,7 @@
|
|||
[flake8]
|
||||
# Recommend matching the black line length (default 88),
|
||||
# rather than using the flake8 default of 79:
|
||||
max-line-length = 88
|
||||
extend-ignore =
|
||||
# See https://github.com/PyCQA/pycodestyle/issues/373
|
||||
E203,
|
||||
|
|
@ -9,6 +9,7 @@ python = "^3.9"
|
|||
tornado = "^6.1"
|
||||
aiosqlite = "^0.17.0"
|
||||
click = "^8.0.1"
|
||||
humanize = "^3.10.0"
|
||||
httpx = "^0.18.2"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
|
|
@ -16,6 +17,9 @@ pytest = "^5.2"
|
|||
black = "^21.6b0"
|
||||
mypy = "^0.910"
|
||||
|
||||
[tool.poetry.scripts]
|
||||
tutor = 'tutor.__main__:cli'
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry>=0.12"]
|
||||
build-backend = "poetry.masonry.api"
|
||||
|
|
|
|||
46
tables.sql
46
tables.sql
|
|
@ -1,7 +1,43 @@
|
|||
CREATE TABLE IF NOT EXISTS `copies` (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
scryfallId TEXT,
|
||||
isFoil INTEGER NOT NULL DEFAULT 0,
|
||||
language TEXT,
|
||||
condition TEXT
|
||||
`id` INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
`scryfall_id` TEXT,
|
||||
`isFoil` INTEGER NOT NULL DEFAULT 0,
|
||||
`language` TEXT,
|
||||
`condition` TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `cards` (
|
||||
`scryfall_id` TEXT PRIMARY KEY,
|
||||
`name` TEXT NOT NULL,
|
||||
`set_code` TEXT NOT NULL,
|
||||
`collector_number` TEXT NOT NULL,
|
||||
`rarity` TEXT NOT NULL,
|
||||
`color_identity` TEXT NOT NULL,
|
||||
`foil` INTEGER NOT NULL DEFAULT 0,
|
||||
`nonfoil` INTEGER NOT NULL DEFAULT 1,
|
||||
`variation` INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `sets` (
|
||||
`set_code` TEXT PRIMARY KEY,
|
||||
`name` TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `rarities` (
|
||||
`rarity` TEXT PRIMARY KEY,
|
||||
`rarity_ord` INTEGER NOT NULL
|
||||
);
|
||||
|
||||
DELETE FROM `rarities`;
|
||||
INSERT INTO `rarities` (`rarity`, `rarity_ord`) VALUES
|
||||
('common', 1),
|
||||
('uncommon', 2),
|
||||
('rare', 3),
|
||||
('special', 4),
|
||||
('mythic', 5),
|
||||
('bonus', 6);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `vars` (
|
||||
`key` TEXT PRIMARY KEY,
|
||||
`value` TEXT
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1,69 +1,17 @@
|
|||
import dataclasses
|
||||
import json
|
||||
import logging
|
||||
import typing
|
||||
|
||||
import aiosqlite
|
||||
import click
|
||||
import httpx
|
||||
import humanize
|
||||
import tornado.ioloop
|
||||
import tornado.web
|
||||
|
||||
import tutor.csvimport
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Card:
|
||||
uuid: str
|
||||
name: str
|
||||
set_code: str
|
||||
|
||||
|
||||
async def search(
|
||||
db: aiosqlite.Connection,
|
||||
name: typing.Optional[str],
|
||||
) -> typing.List[Card]:
|
||||
db.row_factory = aiosqlite.Row
|
||||
constraints = []
|
||||
params = {}
|
||||
if name:
|
||||
constraints.append("cards.name LIKE :name")
|
||||
params["name"] = f"%{name}%"
|
||||
query = " ".join(
|
||||
[
|
||||
"SELECT * FROM cards",
|
||||
"WHERE" if constraints else "",
|
||||
" AND ".join(constraints),
|
||||
"LIMIT 10",
|
||||
]
|
||||
)
|
||||
cursor = await db.execute(query, params)
|
||||
rows = await cursor.fetchall()
|
||||
return [
|
||||
Card(
|
||||
uuid=row["uuid"],
|
||||
name=row["name"],
|
||||
set_code=row["setCode"],
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
class SearchHandler(tornado.web.RequestHandler):
|
||||
async def get(self) -> None:
|
||||
async with aiosqlite.connect("/media/correlr/Correl/AllPrintings.sqlite") as db:
|
||||
cards = await search(db, name=self.get_argument("name", None))
|
||||
self.write(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"uuid": card.uuid,
|
||||
"name": card.name,
|
||||
"set_code": card.set_code,
|
||||
}
|
||||
for card in cards
|
||||
]
|
||||
)
|
||||
)
|
||||
import tutor.database
|
||||
import tutor.scryfall
|
||||
import tutor.server
|
||||
|
||||
|
||||
@click.group()
|
||||
|
|
@ -91,15 +39,12 @@ def cli(ctx, database, log_level):
|
|||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--port", type=int, default=8888)
|
||||
@click.option("--debug", is_flag=True)
|
||||
@click.pass_context
|
||||
def server(ctx):
|
||||
app = tornado.web.Application(
|
||||
[
|
||||
(r"/", SearchHandler),
|
||||
],
|
||||
debug=True,
|
||||
)
|
||||
app.listen(8888)
|
||||
def server(ctx, port, debug):
|
||||
app = tutor.server.make_app({**ctx.obj, "debug": debug})
|
||||
app.listen(port)
|
||||
tornado.ioloop.IOLoop.current().start()
|
||||
|
||||
|
||||
|
|
@ -107,10 +52,54 @@ def server(ctx):
|
|||
@click.argument("filename", type=click.Path(dir_okay=False))
|
||||
@click.pass_context
|
||||
def import_cards(ctx, filename):
|
||||
cards = tornado.ioloop.IOLoop.current().run_sync(
|
||||
tornado.ioloop.IOLoop.current().run_sync(
|
||||
lambda: tutor.csvimport.load(ctx.obj, filename)
|
||||
)
|
||||
|
||||
|
||||
@cli.command("update_scryfall")
|
||||
@click.option("--filename", type=click.Path(dir_okay=False))
|
||||
@click.pass_context
|
||||
def update_scryfall(ctx, filename):
|
||||
if filename:
|
||||
with open(filename) as f:
|
||||
cards = json.loads(f.read())
|
||||
else:
|
||||
response = httpx.get("https://api.scryfall.com/bulk-data/oracle_cards")
|
||||
info = response.json()
|
||||
|
||||
buffer = b""
|
||||
with httpx.stream("GET", info["download_uri"]) as response:
|
||||
downloaded = response.num_bytes_downloaded
|
||||
total = int(response.headers["Content-Length"])
|
||||
with click.progressbar(
|
||||
length=total,
|
||||
label=f"Downloading {humanize.naturalsize(total)}"
|
||||
" of card data from Scryfall",
|
||||
) as bar:
|
||||
for chunk in response.iter_bytes():
|
||||
buffer += chunk
|
||||
bar.update(response.num_bytes_downloaded - downloaded)
|
||||
downloaded = response.num_bytes_downloaded
|
||||
cards = json.loads(buffer)
|
||||
|
||||
async def import_cards():
|
||||
async with aiosqlite.connect(ctx.obj["database"]) as db:
|
||||
with click.progressbar(
|
||||
cards,
|
||||
label=f"Importing {humanize.intcomma(len(cards))} cards",
|
||||
) as bar:
|
||||
for card_object in bar:
|
||||
await tutor.database.store_card(
|
||||
db, tutor.scryfall.to_card(card_object)
|
||||
)
|
||||
await tutor.database.store_set(
|
||||
db, card_object["set"].upper(), card_object["set_name"]
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
tornado.ioloop.IOLoop.current().run_sync(import_cards)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
|
|
|
|||
|
|
@ -3,35 +3,11 @@ import logging
|
|||
import typing
|
||||
|
||||
import aiosqlite
|
||||
import httpx
|
||||
|
||||
import tutor.database
|
||||
import tutor.models
|
||||
|
||||
|
||||
async def find_by_scryfall_id(
|
||||
settings: dict, scryfall_id: str
|
||||
) -> typing.List[tutor.models.Card]:
|
||||
async with aiosqlite.connect(settings["database"]) as db:
|
||||
found = await tutor.database.search(db, scryfall_id=row["Scryfall ID"])
|
||||
if not found:
|
||||
# Scryfall IDs could be pointing to a card in another language, which
|
||||
# won't be in our database. Let's get the card info from Scryfall and
|
||||
# search with that.
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = client.get(f"https://api.scryfall.com/card/{scryfall_id}")
|
||||
if response.status_code != httpx.codes.OK:
|
||||
logging.error("Error response from Scryfall: {response}")
|
||||
return []
|
||||
info = response.json()
|
||||
found = await tutor.database.search(
|
||||
db,
|
||||
collector_number=info["collector_number"],
|
||||
set_code=info["set"].upper(),
|
||||
)
|
||||
return found
|
||||
|
||||
|
||||
async def load(settings: dict, filename: str) -> typing.List[tutor.models.Card]:
|
||||
"""Load cards from a CSV file.
|
||||
|
||||
|
|
|
|||
|
|
@ -25,29 +25,27 @@ async def search(
|
|||
constraints.append("cards.name LIKE :name")
|
||||
params["name"] = name
|
||||
if collector_number is not None:
|
||||
constraints.append("cards.number LIKE :number")
|
||||
constraints.append("cards.collector_number LIKE :number")
|
||||
params["number"] = collector_number
|
||||
if set_code is not None:
|
||||
constraints.append("cards.setCode LIKE :set_code")
|
||||
params["set_code"] = set_code
|
||||
constraints.append("cards.set_code LIKE :set_code")
|
||||
params["set_code"] = set_code.upper()
|
||||
if set_name is not None:
|
||||
constraints.append("sets.name LIKE :set_name")
|
||||
params["set_name"] = set_name
|
||||
if foil is not None:
|
||||
constraints.append("cards.hasFoil IS :foil")
|
||||
constraints.append("cards.foil IS :foil")
|
||||
params["foil"] = foil
|
||||
if alternate_art is not None:
|
||||
constraints.append("cards.isAlternative IS :alternative")
|
||||
constraints.append("cards.variation IS :alternative")
|
||||
params["alternative"] = alternate_art
|
||||
if scryfall_id is not None:
|
||||
constraints.append("cards.scryfallId LIKE :scryfall_id")
|
||||
constraints.append("cards.scryfall_id LIKE :scryfall_id")
|
||||
params["scryfall_id"] = scryfall_id
|
||||
if distinct:
|
||||
constraints.append("(cards.side IS NULL OR cards.side IS 'a')")
|
||||
query = " ".join(
|
||||
[
|
||||
"SELECT cards.* FROM cards",
|
||||
"JOIN sets ON (cards.setCode = sets.code)",
|
||||
"JOIN sets USING (set_code)",
|
||||
"WHERE" if constraints else "",
|
||||
" AND ".join(constraints),
|
||||
f"LIMIT {limit}",
|
||||
|
|
@ -57,18 +55,53 @@ async def search(
|
|||
rows = await cursor.fetchall()
|
||||
return [
|
||||
tutor.models.Card(
|
||||
scryfall_id=uuid.UUID(row["scryfallId"]),
|
||||
scryfall_id=uuid.UUID(row["scryfall_id"]),
|
||||
name=row["name"],
|
||||
set_code=row["setCode"],
|
||||
collector_number=row["number"],
|
||||
set_code=row["set_code"],
|
||||
collector_number=row["collector_number"],
|
||||
rarity=tutor.models.Rarity.from_string(row["rarity"]),
|
||||
color_identity=tutor.models.Color.from_string(row["color_identity"]),
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
async def store_card(db: aiosqlite.Connection, card: tutor.models.Card) -> None:
|
||||
await db.execute(
|
||||
"INSERT INTO cards "
|
||||
"(`scryfall_id`, `name`, `set_code`, `collector_number`, `rarity`,"
|
||||
" `color_identity`) "
|
||||
"VALUES (:scryfall_id, :name, :set_code, :collector_number, :rarity,"
|
||||
" :color_identity) "
|
||||
"ON CONFLICT (scryfall_id) DO UPDATE "
|
||||
"SET `name` = :name,"
|
||||
" `set_code` = :set_code,"
|
||||
" `collector_number` = :collector_number,"
|
||||
" `rarity` = :rarity,"
|
||||
" `color_identity` = :color_identity",
|
||||
{
|
||||
"scryfall_id": str(card.scryfall_id),
|
||||
"name": card.name,
|
||||
"set_code": card.set_code,
|
||||
"collector_number": card.collector_number,
|
||||
"rarity": str(card.rarity),
|
||||
"color_identity": tutor.models.Color.to_string(card.color_identity),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def store_set(db: aiosqlite.Connection, set_code: str, name: str) -> None:
|
||||
await db.execute(
|
||||
"INSERT INTO `sets` (`set_code`, `name`) "
|
||||
"VALUES (:set_code, :name) "
|
||||
"ON CONFLICT (`set_code`) DO NOTHING",
|
||||
{"set_code": set_code, "name": name},
|
||||
)
|
||||
|
||||
|
||||
async def store_copy(db: aiosqlite.Connection, copy: tutor.models.CardCopy) -> None:
|
||||
await db.execute(
|
||||
"INSERT INTO copies (scryfallId, isFoil, condition)"
|
||||
"INSERT INTO copies (scryfall_id, isFoil, condition)"
|
||||
"VALUES (:scryfall_id, :foil, :condition)",
|
||||
{
|
||||
"scryfall_id": str(copy.card.scryfall_id),
|
||||
|
|
|
|||
|
|
@ -1,13 +1,60 @@
|
|||
import dataclasses
|
||||
import enum
|
||||
import typing
|
||||
import uuid
|
||||
|
||||
|
||||
class Color(enum.IntEnum):
|
||||
White = 1
|
||||
Blue = 2
|
||||
Black = 3
|
||||
Green = 4
|
||||
Red = 5
|
||||
|
||||
def __str__(self) -> str:
|
||||
return dict(zip(Color, "WUBGR")).get(self.value)
|
||||
|
||||
@staticmethod
|
||||
def to_string(colors: typing.List["Color"]) -> str:
|
||||
return "".join(map(str, sorted(colors)))
|
||||
|
||||
@staticmethod
|
||||
def from_string(colors: str) -> typing.List["Color"]:
|
||||
return [
|
||||
color
|
||||
for color in [dict(zip("WUBGR", Color)).get(c) for c in colors.upper()]
|
||||
if color is not None
|
||||
]
|
||||
|
||||
|
||||
class Rarity(enum.IntEnum):
|
||||
Common = 1
|
||||
Uncommon = 2
|
||||
Rare = 3
|
||||
Special = 4
|
||||
Mythic = 5
|
||||
Bonus = 6
|
||||
|
||||
def __str__(self):
|
||||
return self.name.lower()
|
||||
|
||||
@staticmethod
|
||||
def from_string(rarity: str) -> "Rarity":
|
||||
try:
|
||||
return {r.name: r for r in Rarity}[rarity.capitalize()]
|
||||
except KeyError:
|
||||
raise TypeError(f"Unknown rarity '{rarity}'")
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Card:
|
||||
scryfall_id: uuid.UUID
|
||||
name: str
|
||||
set_code: str
|
||||
collector_number: str
|
||||
rarity: str
|
||||
color_identity: typing.List[Color]
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class CardCopy:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
import tutor.models
|
||||
|
||||
|
||||
def to_card(data: dict) -> tutor.models.Card:
|
||||
return tutor.models.Card(
|
||||
scryfall_id=data["id"],
|
||||
name=data["name"],
|
||||
set_code=data["set"].upper(),
|
||||
collector_number=data["collector_number"],
|
||||
rarity=tutor.models.Rarity.from_string(data["rarity"]),
|
||||
color_identity=tutor.models.Color.from_string("".join(data["color_identity"])),
|
||||
)
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
import json
|
||||
|
||||
import aiosqlite
|
||||
import tornado.web
|
||||
|
||||
import tutor.database
|
||||
import tutor.models
|
||||
|
||||
|
||||
class SearchHandler(tornado.web.RequestHandler):
|
||||
async def get(self) -> None:
|
||||
async with aiosqlite.connect(self.application.settings["database"]) as db:
|
||||
cards = await tutor.database.search(
|
||||
db, name=self.get_argument("name", None)
|
||||
)
|
||||
self.set_header("Content-Type", "application/json")
|
||||
self.write(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"scryfall_id": str(card.scryfall_id),
|
||||
"name": card.name,
|
||||
"set_code": card.set_code,
|
||||
"collector_number": card.collector_number,
|
||||
"rarity": card.rarity,
|
||||
"color_identity": tutor.models.Color.to_string(
|
||||
card.color_identity
|
||||
),
|
||||
}
|
||||
for card in cards
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def make_app(settings) -> tornado.web.Application:
|
||||
return tornado.web.Application(
|
||||
[
|
||||
(r"/", SearchHandler),
|
||||
],
|
||||
**settings,
|
||||
)
|
||||
Loading…
Reference in New Issue