fixes for foreignNames and readme

README.md

@@ -6,4 +6,21 @@ The goal of this project is to create a SQLite database with all Magic: The Gath
 
 We don't like being dependent on Gatherer for card data, as their update time is always delayed and there are sometimes obvious and egregious mistakes that we can correct in real time.
 
-The database starts as a copy of the MTGJSON source of AllSets-x.json, then we manually go in to correct any errors, as MTGJSON is a direct parse of Gatherer. When spoiler season comes around, we can manually update this (with a tool to be developed later) so we will always have a database that's up to date. Projects can pull in our complete database for their projects in order to have a full Magic: the Gathering card database!
+The database starts as a copy of the MTGJSON source of AllSets-x.json, then we manually go in to correct any errors, as MTGJSON is a direct parse of Gatherer. When spoiler season comes around, we can manually update this (with a tool to be developed later) so we will always have a database that's up to date. Projects can pull in our complete database for their projects in order to have a full Magic: the Gathering card database!
+
+### How to Operate
+
+To turn your JSON file into a SQLite file:
+`./json_to_sql.py create_new_db db_location json_file_location`
+Where:
+* create_new_db is 0 (no) or 1 (yes)
+* db_location is location where the database is OR where you want to store the newly created database
+* json_file_location is the location of the JSON file you want to import to your database
+
+To turn your SQLite file into a JSON file:
+`./sql_to_json.py db_location output_file_location`
+Where:
+* db_location is the location where the database is
+* output_file_location is where you want to store the newly created JSON file
+
+You can test to make sure your initial JSON is the same as your output JSON via `testing_mac.sh`

json_to_sql.py

@@ -9,15 +9,29 @@ import re
 def getVal(data, field):
     val = data.get(field)
     if val:
-        return str(val)
+        return str(val).replace('”', '"').replace('“', '"').replace('’', "'").replace("\\", "")
     return val
     
+def fixJson_foreign(data):
+    if data:
+        data = data.replace("'language'", '"language"')
+        data = data.replace("'multiverseid'", '"multiverseid"')
+        data = data.replace("'name'", '"name"')
+        data = data.replace("', \"", '", "')
+        data = data.replace("\": '", "\": \"")
+        data = data.replace("'}", '"}')
+    return data
+
 def fixJson(data):
     if data:
-        p = re.compile("[A-Za-z]'[A-Za-z]")
+        p = re.compile("[\w]'[\w]")
         for m in p.finditer(data):
             data = data.replace(m.group(), m.group().replace("'", "TMP_HOLD"))
-        data = data.replace("'", '"').replace("TMP_HOLD", "\'")
+
+        p = re.compile("[\w]\"[\w]")
+        for m in p.finditer(data):
+            data = data.replace(m.group(), m.group().replace("'", "DREAK_HOLD"))
+        data = data.replace("'", '"').replace("TMP_HOLD", "'").replace("DREAK_HOLD", '"')
     return data
 
 def create_db(database_connection):
@@ -76,8 +90,8 @@ def json_to_db(json_file_opened, database_connection):
             reserved = getVal(thisCard, "reserved")
             releaseDate = getVal(thisCard, "releaseDate")
             starter = getVal(thisCard, "starter")
-            rulings = getVal(thisCard, "rulings")
+            rulings = ( getVal(thisCard, "rulings") )
-            foreignNames = getVal(thisCard, "foreignNames")
+            foreignNames = fixJson_foreign( getVal(thisCard, "foreignNames") )
             printings = fixJson( getVal(thisCard, "printings") )
             originalText = getVal(thisCard, "originalText")
             originalType = getVal(thisCard, "originalType")
@@ -94,10 +108,14 @@ def json_to_db(json_file_opened, database_connection):
 def main():
     i = sys.argv[1] # Should create new DB 
     d = os.path.expanduser(sys.argv[2]) # File location for database
-    d = sqlite3.connect(d)
     
     if (i == '1'):
+        if os.path.isfile(d):
+            os.remove(d)
+        d = sqlite3.connect(d)
         create_db(d)
+    else:
+        d = sqlite3.connect(d)
         
     xml = os.path.expanduser(sys.argv[3]) # File location for input file
     xml = json.load(open(xml, 'r'))

sql_to_json.py

@@ -71,13 +71,8 @@ def main():
     with open(xml) as f:
         with open(xml2, 'w') as f2:
             for line in f.readlines():
+                # These still need proper parsing
                 if '"rulings":' in line: continue
-                if '"foreignNames":' in line: continue
-                if '"printings":' in line: continue
-                if '"originalText":' in line: continue
-                if '"originalType":' in line: continue
-                if '"legalities":' in line: continue
-                if '"source":' in line: continue
 
                 if any(s in line for s in skip_strings): continue
                 elif any(s in line for s in int_strings):
@@ -92,13 +87,11 @@ def main():
                 if replace_and_write_these_keys(f2, line, "types"): continue
                 if replace_and_write_these_keys(f2, line, "names"): continue
                 
-                #if replace_and_write_these_keys(f2, line, "rulings"): continue
+                if replace_and_write_these_keys(f2, line, "rulings"): continue
-                #if replace_and_write_these_keys(f2, line, "foreignNames"): continue
+                if replace_and_write_these_keys(f2, line, "foreignNames"): continue
                 
                 if any(s in line for s in bool_strings):
                     line = str_to_bool(line)
-                    
-                
                 
                 if 'token card' in line:
                     fix_rarity = True
@@ -170,13 +163,20 @@ def str_to_json(line, key_val):
     if '"' + key_val + '":' in line:
         line_index = line.index('"[')
 
-        line = line[:line_index] + line[line_index:].replace('\\"', '"').replace('”', '\\"').replace('“', '\\"').replace('’', "\\'")[1:]
+        if '377105' in line: # This card is just a mess
+            line = line.replace("Schlafender Drache\\\"", "Schlafender Drache_v_v_")
+            line = line[:line_index] + line[line_index:].replace('\\"', '"')[1:]
+            line = line.replace("_v_v_", "\\\"")
+            line = line.replace('Kongming, "Sleeping Dragon"', 'Kongming, \\"Sleeping Dragon\\"')
+        else:
+            line = line[:line_index] + line[line_index:].replace('\\"', '"')[1:]
 
         while line.strip()[-1:] != "]":
             line = line[:-1]
-         
+
         try: line = line[:line_index] + json.dumps(json.loads(line[line_index:]), indent=2)
-        except: line = line
+        except:
+            line = line
             
         if key_val != "types":
             line += ","

testing_mac.sh

@@ -0,0 +1 @@
+./json_to_sql.py 1 ~/Desktop/Real.db ~/Desktop/AllSets-x.json; ./sql_to_json.py ~/Desktop/Real.db ~/Desktop/MyOutput.json; ./json_to_sql.py 1 ~/Desktop/Mine.db ~/Desktop/MyOutput.json; ./sql_to_json.py ~/Desktop/Mine.db ~/Desktop/MySecondOutput.json; diff ~/Desktop/MyOutput.json ~/Desktop/MySecondOutput.json > ~/Desktop/Diff.txt; md5sum ~/Desktop/MyOutput.json ~/Desktop/MySecondOutput.json