mirror of
https://github.com/correl/mtgsqlive.git
synced 2024-12-01 11:09:57 +00:00
final fixes for bool/ints
This commit is contained in:
Zach H
parent
50b032484f
commit
33263cc4e7
2 changed files with 76 additions and 14 deletions
|
|
@ -4,6 +4,7 @@ import sqlite3
|
|||
import time
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
|
||||
def getVal(data, field):
|
||||
val = data.get(field)
|
||||
|
|
@ -13,12 +14,15 @@ def getVal(data, field):
|
|||
|
||||
def fixJson(data):
|
||||
if data:
|
||||
return data.replace("'", '"')
|
||||
p = re.compile("[A-Za-z]'[A-Za-z]")
|
||||
for m in p.finditer(data):
|
||||
data = data.replace(m.group(), m.group().replace("'", "TMP_HOLD"))
|
||||
data = data.replace("'", '"').replace("TMP_HOLD", "\'")
|
||||
return data
|
||||
|
||||
def create_db(database_connection):
|
||||
c = database_connection.cursor()
|
||||
c.execute('create table cards (id, layout, name, names, manaCost, cmc, colors, colorIdentity, type, supertypes, types, subtypes, rarity, text, flavor, artist, number, power, toughness, loyalty, multiverseid, variations, imageName, watermark, border, timeshifted, hand, life, reserved, releaseDate, starter, rulings, foreignNames, printings, originalText, originalType, legalities, source, setName, setCode, setReleaseDate)')
|
||||
c.execute('create table cards (id, layout, name, names, manaCost, cmc, colors, colorIdentity, type, supertypes, types, subtypes, rarity, text, flavor, artist, number, power, toughness, loyalty, multiverseid, variations, imageName, watermark, border, timeshifted, hand, life, reserved, releaseDate, starter, rulings, foreignNames, printings, originalText, originalType, legalities, source, setName, setCode, setReleaseDate, mciNumber)')
|
||||
c.execute('create table lastUpdated (datetime)')
|
||||
database_connection.commit()
|
||||
c.close()
|
||||
|
|
@ -44,7 +48,7 @@ def json_to_db(json_file_opened, database_connection):
|
|||
thisCard_id = getVal(thisCard, "id")
|
||||
layout = getVal(thisCard, "layout")
|
||||
name = getVal(thisCard, "name")
|
||||
names = getVal(thisCard, "names")
|
||||
names = fixJson( getVal(thisCard, "names") )
|
||||
manaCost = getVal(thisCard, "manaCost")
|
||||
cmc = getVal(thisCard, "cmc")
|
||||
colors = fixJson( getVal(thisCard, "colors") )
|
||||
|
|
@ -79,10 +83,11 @@ def json_to_db(json_file_opened, database_connection):
|
|||
originalType = getVal(thisCard, "originalType")
|
||||
legalities = fixJson( getVal(thisCard, "legalities") )
|
||||
source = getVal(thisCard, "source")
|
||||
mciNumber = getVal(thisCard, "mciNumber")
|
||||
|
||||
thisCard_data = [thisCard_id, layout, name, names, manaCost, cmc, colors, colorIdentity, thisCard_type, supertypes, types, subtypes, rarity, text, flavor, artist, number, power, toughness, loyalty, multiverseid, variations, imageName, watermark, border, timeshifted, hand, life, reserved, releaseDate, starter, rulings, foreignNames, printings, originalText, originalType, legalities, source, setName, thisSet, setReleaseDate]
|
||||
thisCard_data = [thisCard_id, layout, name, names, manaCost, cmc, colors, colorIdentity, thisCard_type, supertypes, types, subtypes, rarity, text, flavor, artist, number, power, toughness, loyalty, multiverseid, variations, imageName, watermark, border, timeshifted, hand, life, reserved, releaseDate, starter, rulings, foreignNames, printings, originalText, originalType, legalities, source, setName, thisSet, setReleaseDate, mciNumber]
|
||||
|
||||
c.execute('insert into cards values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', thisCard_data)
|
||||
c.execute('insert into cards values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', thisCard_data)
|
||||
database_connection.commit()
|
||||
c.close()
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,12 @@ def dict_from_row(row):
|
|||
def remove_empty_keys(d):
|
||||
for k in list(d):
|
||||
if not d[k]:
|
||||
del d[k]
|
||||
try:
|
||||
if d[k] == 0:
|
||||
continue
|
||||
del d[k]
|
||||
except:
|
||||
print("BOOBOO")
|
||||
return d
|
||||
|
||||
def db_to_json(database_connection):
|
||||
|
|
@ -57,6 +62,11 @@ def main():
|
|||
writeFile.write(json_code)
|
||||
writeFile.close()
|
||||
|
||||
int_strings = ('cmc":', 'loyalty":', 'multiverseid":', 'hand":', 'life":')
|
||||
bool_strings = ('reserved":', 'starter":', 'timeshifted":')
|
||||
skip_strings = ('setCode":', 'setName":', 'setReleaseDate":')
|
||||
bonus_comma_strings = ('variations":', 'watermark":')
|
||||
fix_rarity = False
|
||||
# Additional hacks to now cleanup the file, needs to be redone / hopefully not needed
|
||||
with open(xml) as f:
|
||||
with open(xml2, 'w') as f2:
|
||||
|
|
@ -69,7 +79,9 @@ def main():
|
|||
if '"legalities":' in line: continue
|
||||
if '"source":' in line: continue
|
||||
|
||||
|
||||
if any(s in line for s in skip_strings): continue
|
||||
elif any(s in line for s in int_strings):
|
||||
line = str_to_int(line)
|
||||
|
||||
if replace_and_write_these_keys(f2, line, "colorIdentity"): continue
|
||||
if replace_and_write_these_keys(f2, line, "colors"): continue
|
||||
|
|
@ -78,17 +90,61 @@ def main():
|
|||
if replace_and_write_these_keys(f2, line, "subtypes"): continue
|
||||
if replace_and_write_these_keys(f2, line, "legalities"): continue
|
||||
if replace_and_write_these_keys(f2, line, "types"): continue
|
||||
if replace_and_write_these_keys(f2, line, "names"): continue
|
||||
|
||||
#if replace_and_write_these_keys(f2, line, "rulings"): continue
|
||||
#if replace_and_write_these_keys(f2, line, "foreignNames"): continue
|
||||
|
||||
if '"variations":' in line or '"watermark":' in line:
|
||||
if any(s in line for s in bool_strings):
|
||||
line = str_to_bool(line)
|
||||
|
||||
|
||||
|
||||
if 'token card' in line:
|
||||
fix_rarity = True
|
||||
elif fix_rarity and 'rarity":' in line:
|
||||
while line.strip()[-1:] != '"':
|
||||
line = line[:-1]
|
||||
fix_rarity = False
|
||||
|
||||
if any(s in line for s in bonus_comma_strings):
|
||||
f2.write(",")
|
||||
|
||||
if replace_and_write_these_keys(f2, line, "variations"): continue
|
||||
|
||||
f2.write(line)
|
||||
f2.close()
|
||||
cleanup_json(xml2)
|
||||
os.remove(xml)
|
||||
|
||||
def str_to_int(line):
|
||||
try:
|
||||
line_to_int = line.index('": "')
|
||||
line_after_int = line.index('",')
|
||||
except:
|
||||
return line
|
||||
|
||||
try:
|
||||
line = line[:line_to_int] + '": ' + line[line_to_int + 4:line_after_int] + line[line_after_int + 1:]
|
||||
except:
|
||||
print(line_to_int, line_after_int, line)
|
||||
|
||||
return line
|
||||
|
||||
def str_to_bool(line):
|
||||
try:
|
||||
line_to_int = line.index('": "')
|
||||
line_after_int = line.index('",')
|
||||
except:
|
||||
return line
|
||||
|
||||
try:
|
||||
line = line[:line_to_int] + '": ' + line[line_to_int + 4:line_after_int].lower() + line[line_after_int + 1:]
|
||||
except:
|
||||
print(line_to_int, line_after_int, line[line_to_int + 4:line_after_int], line)
|
||||
|
||||
return line
|
||||
|
||||
def cleanup_json(file_path):
|
||||
jsonFile = open(file_path, "r")
|
||||
data = json.load(jsonFile)
|
||||
|
|
@ -101,7 +157,12 @@ def cleanup_json(file_path):
|
|||
def replace_and_write_these_keys(file_opened, line, key_val):
|
||||
retVal = str_to_json(line, key_val)
|
||||
if retVal:
|
||||
file_opened.write(retVal)
|
||||
if key_val != "variations":
|
||||
file_opened.write(retVal)
|
||||
else:
|
||||
while retVal.strip()[-1:] != "]":
|
||||
retVal = retVal[:-1]
|
||||
file_opened.write(retVal)
|
||||
return True
|
||||
|
||||
# Yes this is a mess, but it works for now
|
||||
|
|
@ -114,10 +175,6 @@ def str_to_json(line, key_val):
|
|||
while line.strip()[-1:] != "]":
|
||||
line = line[:-1]
|
||||
|
||||
# BFM causes an issue, manual fix
|
||||
if '"Scariest", "Creature"' in line:
|
||||
line = line.replace('"You"ll"', '"You\'ll"')
|
||||
|
||||
try: line = line[:line_index] + json.dumps(json.loads(line[line_index:]), indent=2)
|
||||
except: line = line
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue