github/mtgsqlive
1
0
Fork 0
mirror of https://github.com/correl/mtgsqlive.git synced 2024-12-01 11:09:57 +00:00
Code Issues Projects Releases Packages Wiki Activity

final fixes for bool/ints

Browse source
This commit is contained in:
Zach H 2016-07-17 01:15:41 -04:00
parent 50b032484f
commit 33263cc4e7
2 changed files with 76 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
json_to_sql.py
View file

@ -4,6 +4,7 @@ import sqlite3
import time
import os
import sys
import re
def getVal(data, field):
val = data.get(field)
@ -13,12 +14,15 @@ def getVal(data, field):
def fixJson(data):
if data:
return data.replace("'", '"')
p = re.compile("[A-Za-z]'[A-Za-z]")
for m in p.finditer(data):
data = data.replace(m.group(), m.group().replace("'", "TMP_HOLD"))
data = data.replace("'", '"').replace("TMP_HOLD", "\'")
return data
def create_db(database_connection):
c = database_connection.cursor()
c.execute('create table cards (id, layout, name, names, manaCost, cmc, colors, colorIdentity, type, supertypes, types, subtypes, rarity, text, flavor, artist, number, power, toughness, loyalty, multiverseid, variations, imageName, watermark, border, timeshifted, hand, life, reserved, releaseDate, starter, rulings, foreignNames, printings, originalText, originalType, legalities, source, setName, setCode, setReleaseDate)')
c.execute('create table cards (id, layout, name, names, manaCost, cmc, colors, colorIdentity, type, supertypes, types, subtypes, rarity, text, flavor, artist, number, power, toughness, loyalty, multiverseid, variations, imageName, watermark, border, timeshifted, hand, life, reserved, releaseDate, starter, rulings, foreignNames, printings, originalText, originalType, legalities, source, setName, setCode, setReleaseDate, mciNumber)')
c.execute('create table lastUpdated (datetime)')
database_connection.commit()
c.close()
@ -44,7 +48,7 @@ def json_to_db(json_file_opened, database_connection):
thisCard_id = getVal(thisCard, "id")
layout = getVal(thisCard, "layout")
name = getVal(thisCard, "name")
names = getVal(thisCard, "names")
names = fixJson( getVal(thisCard, "names") )
manaCost = getVal(thisCard, "manaCost")
cmc = getVal(thisCard, "cmc")
colors = fixJson( getVal(thisCard, "colors") )
@ -79,10 +83,11 @@ def json_to_db(json_file_opened, database_connection):
originalType = getVal(thisCard, "originalType")
legalities = fixJson( getVal(thisCard, "legalities") )
source = getVal(thisCard, "source")
mciNumber = getVal(thisCard, "mciNumber")
thisCard_data = [thisCard_id, layout, name, names, manaCost, cmc, colors, colorIdentity, thisCard_type, supertypes, types, subtypes, rarity, text, flavor, artist, number, power, toughness, loyalty, multiverseid, variations, imageName, watermark, border, timeshifted, hand, life, reserved, releaseDate, starter, rulings, foreignNames, printings, originalText, originalType, legalities, source, setName, thisSet, setReleaseDate]
thisCard_data = [thisCard_id, layout, name, names, manaCost, cmc, colors, colorIdentity, thisCard_type, supertypes, types, subtypes, rarity, text, flavor, artist, number, power, toughness, loyalty, multiverseid, variations, imageName, watermark, border, timeshifted, hand, life, reserved, releaseDate, starter, rulings, foreignNames, printings, originalText, originalType, legalities, source, setName, thisSet, setReleaseDate, mciNumber]
c.execute('insert into cards values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', thisCard_data)
c.execute('insert into cards values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', thisCard_data)
database_connection.commit()
c.close()

69
sql_to_json.py
View file

@ -11,7 +11,12 @@ def dict_from_row(row):
def remove_empty_keys(d):
for k in list(d):
if not d[k]:
try:
if d[k] == 0:
continue
del d[k]
except:
print("BOOBOO")
return d
def db_to_json(database_connection):
@ -57,6 +62,11 @@ def main():
writeFile.write(json_code)
writeFile.close()
int_strings = ('cmc":', 'loyalty":', 'multiverseid":', 'hand":', 'life":')
bool_strings = ('reserved":', 'starter":', 'timeshifted":')
skip_strings = ('setCode":', 'setName":', 'setReleaseDate":')
bonus_comma_strings = ('variations":', 'watermark":')
fix_rarity = False
# Additional hacks to now cleanup the file, needs to be redone / hopefully not needed
with open(xml) as f:
with open(xml2, 'w') as f2:
@ -69,7 +79,9 @@ def main():
if '"legalities":' in line: continue
if '"source":' in line: continue
if any(s in line for s in skip_strings): continue
elif any(s in line for s in int_strings):
line = str_to_int(line)
if replace_and_write_these_keys(f2, line, "colorIdentity"): continue
if replace_and_write_these_keys(f2, line, "colors"): continue
@ -78,17 +90,61 @@ def main():
if replace_and_write_these_keys(f2, line, "subtypes"): continue
if replace_and_write_these_keys(f2, line, "legalities"): continue
if replace_and_write_these_keys(f2, line, "types"): continue
if replace_and_write_these_keys(f2, line, "names"): continue
#if replace_and_write_these_keys(f2, line, "rulings"): continue
#if replace_and_write_these_keys(f2, line, "foreignNames"): continue
if '"variations":' in line or '"watermark":' in line:
if any(s in line for s in bool_strings):
line = str_to_bool(line)
if 'token card' in line:
fix_rarity = True
elif fix_rarity and 'rarity":' in line:
while line.strip()[-1:] != '"':
line = line[:-1]
fix_rarity = False
if any(s in line for s in bonus_comma_strings):
f2.write(",")
if replace_and_write_these_keys(f2, line, "variations"): continue
f2.write(line)
f2.close()
cleanup_json(xml2)
os.remove(xml)
def str_to_int(line):
try:
line_to_int = line.index('": "')
line_after_int = line.index('",')
except:
return line
try:
line = line[:line_to_int] + '": ' + line[line_to_int + 4:line_after_int] + line[line_after_int + 1:]
except:
print(line_to_int, line_after_int, line)
return line
def str_to_bool(line):
try:
line_to_int = line.index('": "')
line_after_int = line.index('",')
except:
return line
try:
line = line[:line_to_int] + '": ' + line[line_to_int + 4:line_after_int].lower() + line[line_after_int + 1:]
except:
print(line_to_int, line_after_int, line[line_to_int + 4:line_after_int], line)
return line
def cleanup_json(file_path):
jsonFile = open(file_path, "r")
data = json.load(jsonFile)
@ -101,6 +157,11 @@ def cleanup_json(file_path):
def replace_and_write_these_keys(file_opened, line, key_val):
retVal = str_to_json(line, key_val)
if retVal:
if key_val != "variations":
file_opened.write(retVal)
else:
while retVal.strip()[-1:] != "]":
retVal = retVal[:-1]
file_opened.write(retVal)
return True
@ -114,10 +175,6 @@ def str_to_json(line, key_val):
while line.strip()[-1:] != "]":
line = line[:-1]
# BFM causes an issue, manual fix
if '"Scariest", "Creature"' in line:
line = line.replace('"You"ll"', '"You\'ll"')
try: line = line[:line_index] + json.dumps(json.loads(line[line_index:]), indent=2)
except: line = line