From 33263cc4e701da58850a0fd309f8ff184f963290 Mon Sep 17 00:00:00 2001 From: Zach H Date: Sun, 17 Jul 2016 01:15:41 -0400 Subject: [PATCH] final fixes for bool/ints --- json_to_sql.py | 15 ++++++---- sql_to_json.py | 75 ++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 76 insertions(+), 14 deletions(-) diff --git a/json_to_sql.py b/json_to_sql.py index 49f7bca..d4ec6aa 100755 --- a/json_to_sql.py +++ b/json_to_sql.py @@ -4,6 +4,7 @@ import sqlite3 import time import os import sys +import re def getVal(data, field): val = data.get(field) @@ -13,12 +14,15 @@ def getVal(data, field): def fixJson(data): if data: - return data.replace("'", '"') + p = re.compile("[A-Za-z]'[A-Za-z]") + for m in p.finditer(data): + data = data.replace(m.group(), m.group().replace("'", "TMP_HOLD")) + data = data.replace("'", '"').replace("TMP_HOLD", "\'") return data def create_db(database_connection): c = database_connection.cursor() - c.execute('create table cards (id, layout, name, names, manaCost, cmc, colors, colorIdentity, type, supertypes, types, subtypes, rarity, text, flavor, artist, number, power, toughness, loyalty, multiverseid, variations, imageName, watermark, border, timeshifted, hand, life, reserved, releaseDate, starter, rulings, foreignNames, printings, originalText, originalType, legalities, source, setName, setCode, setReleaseDate)') + c.execute('create table cards (id, layout, name, names, manaCost, cmc, colors, colorIdentity, type, supertypes, types, subtypes, rarity, text, flavor, artist, number, power, toughness, loyalty, multiverseid, variations, imageName, watermark, border, timeshifted, hand, life, reserved, releaseDate, starter, rulings, foreignNames, printings, originalText, originalType, legalities, source, setName, setCode, setReleaseDate, mciNumber)') c.execute('create table lastUpdated (datetime)') database_connection.commit() c.close() @@ -44,7 +48,7 @@ def json_to_db(json_file_opened, database_connection): thisCard_id = getVal(thisCard, "id") layout = getVal(thisCard, "layout") name = getVal(thisCard, "name") - names = getVal(thisCard, "names") + names = fixJson( getVal(thisCard, "names") ) manaCost = getVal(thisCard, "manaCost") cmc = getVal(thisCard, "cmc") colors = fixJson( getVal(thisCard, "colors") ) @@ -79,10 +83,11 @@ def json_to_db(json_file_opened, database_connection): originalType = getVal(thisCard, "originalType") legalities = fixJson( getVal(thisCard, "legalities") ) source = getVal(thisCard, "source") + mciNumber = getVal(thisCard, "mciNumber") - thisCard_data = [thisCard_id, layout, name, names, manaCost, cmc, colors, colorIdentity, thisCard_type, supertypes, types, subtypes, rarity, text, flavor, artist, number, power, toughness, loyalty, multiverseid, variations, imageName, watermark, border, timeshifted, hand, life, reserved, releaseDate, starter, rulings, foreignNames, printings, originalText, originalType, legalities, source, setName, thisSet, setReleaseDate] + thisCard_data = [thisCard_id, layout, name, names, manaCost, cmc, colors, colorIdentity, thisCard_type, supertypes, types, subtypes, rarity, text, flavor, artist, number, power, toughness, loyalty, multiverseid, variations, imageName, watermark, border, timeshifted, hand, life, reserved, releaseDate, starter, rulings, foreignNames, printings, originalText, originalType, legalities, source, setName, thisSet, setReleaseDate, mciNumber] - c.execute('insert into cards values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', thisCard_data) + c.execute('insert into cards values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', thisCard_data) database_connection.commit() c.close() diff --git a/sql_to_json.py b/sql_to_json.py index 7242073..43a0853 100755 --- a/sql_to_json.py +++ b/sql_to_json.py @@ -11,7 +11,12 @@ def dict_from_row(row): def remove_empty_keys(d): for k in list(d): if not d[k]: - del d[k] + try: + if d[k] == 0: + continue + del d[k] + except: + print("BOOBOO") return d def db_to_json(database_connection): @@ -57,6 +62,11 @@ def main(): writeFile.write(json_code) writeFile.close() + int_strings = ('cmc":', 'loyalty":', 'multiverseid":', 'hand":', 'life":') + bool_strings = ('reserved":', 'starter":', 'timeshifted":') + skip_strings = ('setCode":', 'setName":', 'setReleaseDate":') + bonus_comma_strings = ('variations":', 'watermark":') + fix_rarity = False # Additional hacks to now cleanup the file, needs to be redone / hopefully not needed with open(xml) as f: with open(xml2, 'w') as f2: @@ -68,8 +78,10 @@ def main(): if '"originalType":' in line: continue if '"legalities":' in line: continue if '"source":' in line: continue - - + + if any(s in line for s in skip_strings): continue + elif any(s in line for s in int_strings): + line = str_to_int(line) if replace_and_write_these_keys(f2, line, "colorIdentity"): continue if replace_and_write_these_keys(f2, line, "colors"): continue @@ -78,17 +90,61 @@ def main(): if replace_and_write_these_keys(f2, line, "subtypes"): continue if replace_and_write_these_keys(f2, line, "legalities"): continue if replace_and_write_these_keys(f2, line, "types"): continue + if replace_and_write_these_keys(f2, line, "names"): continue + #if replace_and_write_these_keys(f2, line, "rulings"): continue #if replace_and_write_these_keys(f2, line, "foreignNames"): continue - if '"variations":' in line or '"watermark":' in line: + if any(s in line for s in bool_strings): + line = str_to_bool(line) + + + + if 'token card' in line: + fix_rarity = True + elif fix_rarity and 'rarity":' in line: + while line.strip()[-1:] != '"': + line = line[:-1] + fix_rarity = False + + if any(s in line for s in bonus_comma_strings): f2.write(",") + + if replace_and_write_these_keys(f2, line, "variations"): continue f2.write(line) f2.close() cleanup_json(xml2) os.remove(xml) +def str_to_int(line): + try: + line_to_int = line.index('": "') + line_after_int = line.index('",') + except: + return line + + try: + line = line[:line_to_int] + '": ' + line[line_to_int + 4:line_after_int] + line[line_after_int + 1:] + except: + print(line_to_int, line_after_int, line) + + return line + +def str_to_bool(line): + try: + line_to_int = line.index('": "') + line_after_int = line.index('",') + except: + return line + + try: + line = line[:line_to_int] + '": ' + line[line_to_int + 4:line_after_int].lower() + line[line_after_int + 1:] + except: + print(line_to_int, line_after_int, line[line_to_int + 4:line_after_int], line) + + return line + def cleanup_json(file_path): jsonFile = open(file_path, "r") data = json.load(jsonFile) @@ -101,7 +157,12 @@ def cleanup_json(file_path): def replace_and_write_these_keys(file_opened, line, key_val): retVal = str_to_json(line, key_val) if retVal: - file_opened.write(retVal) + if key_val != "variations": + file_opened.write(retVal) + else: + while retVal.strip()[-1:] != "]": + retVal = retVal[:-1] + file_opened.write(retVal) return True # Yes this is a mess, but it works for now @@ -113,10 +174,6 @@ def str_to_json(line, key_val): while line.strip()[-1:] != "]": line = line[:-1] - - # BFM causes an issue, manual fix - if '"Scariest", "Creature"' in line: - line = line.replace('"You"ll"', '"You\'ll"') try: line = line[:line_index] + json.dumps(json.loads(line[line_index:]), indent=2) except: line = line