mirror of
https://github.com/correl/mtgsqlive.git
synced 2024-12-01 11:09:57 +00:00
fixes for foreignNames and readme
This commit is contained in:
Zach H
parent
33263cc4e7
commit
26c12d4aa3
4 changed files with 56 additions and 20 deletions
19
README.md
19
README.md
|
|
@ -6,4 +6,21 @@ The goal of this project is to create a SQLite database with all Magic: The Gath
|
|||
|
||||
We don't like being dependent on Gatherer for card data, as their update time is always delayed and there are sometimes obvious and egregious mistakes that we can correct in real time.
|
||||
|
||||
The database starts as a copy of the MTGJSON source of AllSets-x.json, then we manually go in to correct any errors, as MTGJSON is a direct parse of Gatherer. When spoiler season comes around, we can manually update this (with a tool to be developed later) so we will always have a database that's up to date. Projects can pull in our complete database for their projects in order to have a full Magic: the Gathering card database!
|
||||
The database starts as a copy of the MTGJSON source of AllSets-x.json, then we manually go in to correct any errors, as MTGJSON is a direct parse of Gatherer. When spoiler season comes around, we can manually update this (with a tool to be developed later) so we will always have a database that's up to date. Projects can pull in our complete database for their projects in order to have a full Magic: the Gathering card database!
|
||||
|
||||
### How to Operate
|
||||
|
||||
To turn your JSON file into a SQLite file:
|
||||
`./json_to_sql.py create_new_db db_location json_file_location`
|
||||
Where:
|
||||
* create_new_db is 0 (no) or 1 (yes)
|
||||
* db_location is location where the database is OR where you want to store the newly created database
|
||||
* json_file_location is the location of the JSON file you want to import to your database
|
||||
|
||||
To turn your SQLite file into a JSON file:
|
||||
`./sql_to_json.py db_location output_file_location`
|
||||
Where:
|
||||
* db_location is the location where the database is
|
||||
* output_file_location is where you want to store the newly created JSON file
|
||||
|
||||
You can test to make sure your initial JSON is the same as your output JSON via `testing_mac.sh`
|
||||
|
|
@ -9,15 +9,29 @@ import re
|
|||
def getVal(data, field):
|
||||
val = data.get(field)
|
||||
if val:
|
||||
return str(val)
|
||||
return str(val).replace('”', '"').replace('“', '"').replace('’', "'").replace("\\", "")
|
||||
return val
|
||||
|
||||
def fixJson_foreign(data):
|
||||
if data:
|
||||
data = data.replace("'language'", '"language"')
|
||||
data = data.replace("'multiverseid'", '"multiverseid"')
|
||||
data = data.replace("'name'", '"name"')
|
||||
data = data.replace("', \"", '", "')
|
||||
data = data.replace("\": '", "\": \"")
|
||||
data = data.replace("'}", '"}')
|
||||
return data
|
||||
|
||||
def fixJson(data):
|
||||
if data:
|
||||
p = re.compile("[A-Za-z]'[A-Za-z]")
|
||||
p = re.compile("[\w]'[\w]")
|
||||
for m in p.finditer(data):
|
||||
data = data.replace(m.group(), m.group().replace("'", "TMP_HOLD"))
|
||||
data = data.replace("'", '"').replace("TMP_HOLD", "\'")
|
||||
|
||||
p = re.compile("[\w]\"[\w]")
|
||||
for m in p.finditer(data):
|
||||
data = data.replace(m.group(), m.group().replace("'", "DREAK_HOLD"))
|
||||
data = data.replace("'", '"').replace("TMP_HOLD", "'").replace("DREAK_HOLD", '"')
|
||||
return data
|
||||
|
||||
def create_db(database_connection):
|
||||
|
|
@ -76,8 +90,8 @@ def json_to_db(json_file_opened, database_connection):
|
|||
reserved = getVal(thisCard, "reserved")
|
||||
releaseDate = getVal(thisCard, "releaseDate")
|
||||
starter = getVal(thisCard, "starter")
|
||||
rulings = getVal(thisCard, "rulings")
|
||||
foreignNames = getVal(thisCard, "foreignNames")
|
||||
rulings = ( getVal(thisCard, "rulings") )
|
||||
foreignNames = fixJson_foreign( getVal(thisCard, "foreignNames") )
|
||||
printings = fixJson( getVal(thisCard, "printings") )
|
||||
originalText = getVal(thisCard, "originalText")
|
||||
originalType = getVal(thisCard, "originalType")
|
||||
|
|
@ -94,10 +108,14 @@ def json_to_db(json_file_opened, database_connection):
|
|||
def main():
|
||||
i = sys.argv[1] # Should create new DB
|
||||
d = os.path.expanduser(sys.argv[2]) # File location for database
|
||||
d = sqlite3.connect(d)
|
||||
|
||||
if (i == '1'):
|
||||
if os.path.isfile(d):
|
||||
os.remove(d)
|
||||
d = sqlite3.connect(d)
|
||||
create_db(d)
|
||||
else:
|
||||
d = sqlite3.connect(d)
|
||||
|
||||
xml = os.path.expanduser(sys.argv[3]) # File location for input file
|
||||
xml = json.load(open(xml, 'r'))
|
||||
|
|
|
|||
|
|
@ -71,13 +71,8 @@ def main():
|
|||
with open(xml) as f:
|
||||
with open(xml2, 'w') as f2:
|
||||
for line in f.readlines():
|
||||
# These still need proper parsing
|
||||
if '"rulings":' in line: continue
|
||||
if '"foreignNames":' in line: continue
|
||||
if '"printings":' in line: continue
|
||||
if '"originalText":' in line: continue
|
||||
if '"originalType":' in line: continue
|
||||
if '"legalities":' in line: continue
|
||||
if '"source":' in line: continue
|
||||
|
||||
if any(s in line for s in skip_strings): continue
|
||||
elif any(s in line for s in int_strings):
|
||||
|
|
@ -92,13 +87,11 @@ def main():
|
|||
if replace_and_write_these_keys(f2, line, "types"): continue
|
||||
if replace_and_write_these_keys(f2, line, "names"): continue
|
||||
|
||||
#if replace_and_write_these_keys(f2, line, "rulings"): continue
|
||||
#if replace_and_write_these_keys(f2, line, "foreignNames"): continue
|
||||
if replace_and_write_these_keys(f2, line, "rulings"): continue
|
||||
if replace_and_write_these_keys(f2, line, "foreignNames"): continue
|
||||
|
||||
if any(s in line for s in bool_strings):
|
||||
line = str_to_bool(line)
|
||||
|
||||
|
||||
|
||||
if 'token card' in line:
|
||||
fix_rarity = True
|
||||
|
|
@ -170,13 +163,20 @@ def str_to_json(line, key_val):
|
|||
if '"' + key_val + '":' in line:
|
||||
line_index = line.index('"[')
|
||||
|
||||
line = line[:line_index] + line[line_index:].replace('\\"', '"').replace('”', '\\"').replace('“', '\\"').replace('’', "\\'")[1:]
|
||||
if '377105' in line: # This card is just a mess
|
||||
line = line.replace("Schlafender Drache\\\"", "Schlafender Drache_v_v_")
|
||||
line = line[:line_index] + line[line_index:].replace('\\"', '"')[1:]
|
||||
line = line.replace("_v_v_", "\\\"")
|
||||
line = line.replace('Kongming, "Sleeping Dragon"', 'Kongming, \\"Sleeping Dragon\\"')
|
||||
else:
|
||||
line = line[:line_index] + line[line_index:].replace('\\"', '"')[1:]
|
||||
|
||||
while line.strip()[-1:] != "]":
|
||||
line = line[:-1]
|
||||
|
||||
|
||||
try: line = line[:line_index] + json.dumps(json.loads(line[line_index:]), indent=2)
|
||||
except: line = line
|
||||
except:
|
||||
line = line
|
||||
|
||||
if key_val != "types":
|
||||
line += ","
|
||||
|
|
|
|||
1
testing_mac.sh
Normal file
1
testing_mac.sh
Normal file
|
|
@ -0,0 +1 @@
|
|||
./json_to_sql.py 1 ~/Desktop/Real.db ~/Desktop/AllSets-x.json; ./sql_to_json.py ~/Desktop/Real.db ~/Desktop/MyOutput.json; ./json_to_sql.py 1 ~/Desktop/Mine.db ~/Desktop/MyOutput.json; ./sql_to_json.py ~/Desktop/Mine.db ~/Desktop/MySecondOutput.json; diff ~/Desktop/MyOutput.json ~/Desktop/MySecondOutput.json > ~/Desktop/Diff.txt; md5sum ~/Desktop/MyOutput.json ~/Desktop/MySecondOutput.json
|
||||
Loading…
Reference in a new issue