Track external ID and use when importing

postgres/005-external-id.sql

@@ -0,0 +1,4 @@
+ALTER TABLE "copies"
+ADD COLUMN IF NOT EXISTS "external_id" TEXT UNIQUE;
+
+CREATE INDEX IF NOT EXISTS "copies_external_id" ON "copies" ("external_id");

tutor/cli.py

@@ -242,12 +242,15 @@ def update_scryfall(ctx, filename):
                         try:
                             card = tutor.scryfall.to_card(card_json).unwrap()
                             await tutor.database.store_set(
-                                cursor, card_json["set"].upper(), card_json["set_name"]
+                                cursor,
+                                card_json["set"].upper(),
+                                card_json["set_type"],
+                                card_json["set_name"],
                             )
                             await tutor.database.store_card(cursor, card)
                             await tutor.database.store_price(cursor, today, card)
-                        except:
+                        except Exception as e:
-                            print(f"Skipping entry {card_json}")
+                            print(f"Skipping entry {card_json}: {e}")
                         bar.update(read)
                 print("Updating oracle card data & indexes")
                 await cursor.execute(

tutor/csvimport.py

@@ -36,6 +36,7 @@ async def load(
             for row in reader:
                 is_foil = "Foil" in row and row["Foil"].lower() == "foil"
                 quantity = int(row.get("Quantity", row.get("Count", 1)))
+                external_id = row.get("Delver Local ID")
                 if "Creation Date" in row:
                     created_date = arrow.get(
                         row["Creation Date"].replace("_", " ")
@@ -68,6 +69,7 @@ async def load(
                         language=row["Language"] or "English",
                         collection=row.get("List name") or "Default",
                         created_date=created_date,
+                        external_id=external_id,
                     )
                     logging.info((quantity, card))
                     for i in range(quantity):

tutor/database.py

@@ -351,8 +351,26 @@ async def store_set(
 async def store_copy(db: psycopg.AsyncCursor, copy: tutor.models.CardCopy) -> None:
     await db.execute(
         """
-        INSERT INTO copies ("scryfall_id", "isFoil", "collection", "condition", "created_date")
+        INSERT INTO copies ( "scryfall_id"
-        VALUES (%(scryfall_id)s, %(foil)s, %(collection)s, %(condition)s, %(created_date)s)
+                           , "isFoil"
+                           , "collection"
+                           , "condition"
+                           , "created_date"
+                           , "external_id"
+                           )
+        VALUES ( %(scryfall_id)s
+               , %(foil)s
+               , %(collection)s
+               , %(condition)s
+               , %(created_date)s
+               , %(external_id)s
+               )
+        ON CONFLICT ("external_id") DO UPDATE
+        SET "scryfall_id" = %(scryfall_id)s
+          , "isFoil" = %(foil)s
+          , "collection" = %(collection)s
+          , "condition" = %(condition)s
+          , "created_date" = %(created_date)s
         """,
         {
             "scryfall_id": str(copy.card.scryfall_id),
@@ -360,6 +378,7 @@ async def store_copy(db: psycopg.AsyncCursor, copy: tutor.models.CardCopy) -> No
             "collection": copy.collection,
             "condition": copy.condition,
             "created_date": str(copy.created_date),
+            "external_id": copy.external_id,
         },
     )
 

tutor/models.py

@@ -108,6 +108,7 @@ class CardCopy:
     foil: bool
     language: str = "English"
     collection: str = "Default"
+    external_id: typing.Optional[str] = None
     condition: typing.Optional[str] = None
     created_date: datetime.datetime = dataclasses.field(
         default_factory=datetime.datetime.now