diff --git a/README.md b/README.md index 282ac3e..cc708da 100644 --- a/README.md +++ b/README.md @@ -6,4 +6,21 @@ The goal of this project is to create a SQLite database with all Magic: The Gath We don't like being dependent on Gatherer for card data, as their update time is always delayed and there are sometimes obvious and egregious mistakes that we can correct in real time. -The database starts as a copy of the MTGJSON source of AllSets-x.json, then we manually go in to correct any errors, as MTGJSON is a direct parse of Gatherer. When spoiler season comes around, we can manually update this (with a tool to be developed later) so we will always have a database that's up to date. Projects can pull in our complete database for their projects in order to have a full Magic: the Gathering card database! \ No newline at end of file +The database starts as a copy of the MTGJSON source of AllSets-x.json, then we manually go in to correct any errors, as MTGJSON is a direct parse of Gatherer. When spoiler season comes around, we can manually update this (with a tool to be developed later) so we will always have a database that's up to date. Projects can pull in our complete database for their projects in order to have a full Magic: the Gathering card database! + +### How to Operate + +To turn your JSON file into a SQLite file: +`./json_to_sql.py create_new_db db_location json_file_location` +Where: +* create_new_db is 0 (no) or 1 (yes) +* db_location is location where the database is OR where you want to store the newly created database +* json_file_location is the location of the JSON file you want to import to your database + +To turn your SQLite file into a JSON file: +`./sql_to_json.py db_location output_file_location` +Where: +* db_location is the location where the database is +* output_file_location is where you want to store the newly created JSON file + +You can test to make sure your initial JSON is the same as your output JSON via `testing_mac.sh` \ No newline at end of file diff --git a/json_to_sql.py b/json_to_sql.py index d4ec6aa..9206948 100755 --- a/json_to_sql.py +++ b/json_to_sql.py @@ -9,15 +9,29 @@ import re def getVal(data, field): val = data.get(field) if val: - return str(val) + return str(val).replace('”', '"').replace('“', '"').replace('’', "'").replace("\\", "") return val +def fixJson_foreign(data): + if data: + data = data.replace("'language'", '"language"') + data = data.replace("'multiverseid'", '"multiverseid"') + data = data.replace("'name'", '"name"') + data = data.replace("', \"", '", "') + data = data.replace("\": '", "\": \"") + data = data.replace("'}", '"}') + return data + def fixJson(data): if data: - p = re.compile("[A-Za-z]'[A-Za-z]") + p = re.compile("[\w]'[\w]") for m in p.finditer(data): data = data.replace(m.group(), m.group().replace("'", "TMP_HOLD")) - data = data.replace("'", '"').replace("TMP_HOLD", "\'") + + p = re.compile("[\w]\"[\w]") + for m in p.finditer(data): + data = data.replace(m.group(), m.group().replace("'", "DREAK_HOLD")) + data = data.replace("'", '"').replace("TMP_HOLD", "'").replace("DREAK_HOLD", '"') return data def create_db(database_connection): @@ -76,8 +90,8 @@ def json_to_db(json_file_opened, database_connection): reserved = getVal(thisCard, "reserved") releaseDate = getVal(thisCard, "releaseDate") starter = getVal(thisCard, "starter") - rulings = getVal(thisCard, "rulings") - foreignNames = getVal(thisCard, "foreignNames") + rulings = ( getVal(thisCard, "rulings") ) + foreignNames = fixJson_foreign( getVal(thisCard, "foreignNames") ) printings = fixJson( getVal(thisCard, "printings") ) originalText = getVal(thisCard, "originalText") originalType = getVal(thisCard, "originalType") @@ -94,10 +108,14 @@ def json_to_db(json_file_opened, database_connection): def main(): i = sys.argv[1] # Should create new DB d = os.path.expanduser(sys.argv[2]) # File location for database - d = sqlite3.connect(d) if (i == '1'): + if os.path.isfile(d): + os.remove(d) + d = sqlite3.connect(d) create_db(d) + else: + d = sqlite3.connect(d) xml = os.path.expanduser(sys.argv[3]) # File location for input file xml = json.load(open(xml, 'r')) diff --git a/sql_to_json.py b/sql_to_json.py index 43a0853..768692c 100755 --- a/sql_to_json.py +++ b/sql_to_json.py @@ -71,13 +71,8 @@ def main(): with open(xml) as f: with open(xml2, 'w') as f2: for line in f.readlines(): + # These still need proper parsing if '"rulings":' in line: continue - if '"foreignNames":' in line: continue - if '"printings":' in line: continue - if '"originalText":' in line: continue - if '"originalType":' in line: continue - if '"legalities":' in line: continue - if '"source":' in line: continue if any(s in line for s in skip_strings): continue elif any(s in line for s in int_strings): @@ -92,13 +87,11 @@ def main(): if replace_and_write_these_keys(f2, line, "types"): continue if replace_and_write_these_keys(f2, line, "names"): continue - #if replace_and_write_these_keys(f2, line, "rulings"): continue - #if replace_and_write_these_keys(f2, line, "foreignNames"): continue + if replace_and_write_these_keys(f2, line, "rulings"): continue + if replace_and_write_these_keys(f2, line, "foreignNames"): continue if any(s in line for s in bool_strings): line = str_to_bool(line) - - if 'token card' in line: fix_rarity = True @@ -170,13 +163,20 @@ def str_to_json(line, key_val): if '"' + key_val + '":' in line: line_index = line.index('"[') - line = line[:line_index] + line[line_index:].replace('\\"', '"').replace('”', '\\"').replace('“', '\\"').replace('’', "\\'")[1:] + if '377105' in line: # This card is just a mess + line = line.replace("Schlafender Drache\\\"", "Schlafender Drache_v_v_") + line = line[:line_index] + line[line_index:].replace('\\"', '"')[1:] + line = line.replace("_v_v_", "\\\"") + line = line.replace('Kongming, "Sleeping Dragon"', 'Kongming, \\"Sleeping Dragon\\"') + else: + line = line[:line_index] + line[line_index:].replace('\\"', '"')[1:] while line.strip()[-1:] != "]": line = line[:-1] - + try: line = line[:line_index] + json.dumps(json.loads(line[line_index:]), indent=2) - except: line = line + except: + line = line if key_val != "types": line += "," diff --git a/testing_mac.sh b/testing_mac.sh new file mode 100644 index 0000000..b0fc1bd --- /dev/null +++ b/testing_mac.sh @@ -0,0 +1 @@ +./json_to_sql.py 1 ~/Desktop/Real.db ~/Desktop/AllSets-x.json; ./sql_to_json.py ~/Desktop/Real.db ~/Desktop/MyOutput.json; ./json_to_sql.py 1 ~/Desktop/Mine.db ~/Desktop/MyOutput.json; ./sql_to_json.py ~/Desktop/Mine.db ~/Desktop/MySecondOutput.json; diff ~/Desktop/MyOutput.json ~/Desktop/MySecondOutput.json > ~/Desktop/Diff.txt; md5sum ~/Desktop/MyOutput.json ~/Desktop/MySecondOutput.json \ No newline at end of file