Compose DB out of data stored externally

Author: DarkFenX

.appveyor.yml

@@ -65,6 +65,8 @@ before_build:
 build_script:
   - ECHO "Build pyfa:"
 
+  # Build gamedata DB
+  - "python db_update.py"
   ##########
   # PyInstaller - create binaries for pyfa
   ##########

.travis.yml

@@ -13,6 +13,7 @@ matrix:
 before_install:
   - bash scripts/setup-osx.sh
 install:
+  - python3 db_update.py
   - export PYFA_VERSION="$(python3 scripts/dump_version.py)"
   - bash scripts/package-osx.sh
 before_deploy:

scripts/jsonToSql.py db_update.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #======================================================================
 # Copyright (C) 2012 Diego Duclos
 #
@@ -18,35 +18,65 @@
 # License along with eos.  If not, see <http://www.gnu.org/licenses/>.
 #======================================================================
 
+
 import functools
+import itertools
+import json
 import os
+import sqlite3
 import sys
 
-# Add eos root path to sys.path so we can import ourselves
-path = os.path.dirname(__file__)
-sys.path.insert(0, os.path.realpath(os.path.join(path, '..')))
 
-import json
-import argparse
-import itertools
+ROOT_DIR = os.path.realpath(os.path.dirname(__file__))
+DB_PATH = os.path.join(ROOT_DIR, 'eve.db')
+JSON_DIR = os.path.join(ROOT_DIR, 'staticdata')
+if ROOT_DIR not in sys.path:
+    sys.path.insert(0, ROOT_DIR)
+GAMEDATA_SCHEMA_VERSION = 1
 
 
 CATEGORIES_TO_REMOVE = [
     30  # Apparel
 ]
 
-def main(db, json_path):
-    if os.path.isfile(db):
-        os.remove(db)
 
-    jsonPath = os.path.expanduser(json_path)
+def db_needs_update():
+    """True if needs, false if it does not, none if we cannot check it."""
+    try:
+        with open(os.path.join(JSON_DIR, 'phobos', 'metadata.json')) as f:
+            data_version = next((r['field_value'] for r in json.load(f) if r['field_name'] == 'client_build'))
+    except KeyboardInterrupt:
+        raise
+    # If we have no source data - return None; should not update in this case
+    except:
+        return None
+    if not os.path.isfile(DB_PATH):
+        return True
+    db_data_version = None
+    db_schema_version = None
+    try:
+        db = sqlite3.connect(DB_PATH)
+        cursor = db.cursor()
+        cursor.execute('SELECT field_value FROM metadata WHERE field_name = \'client_build\'')
+        for row in cursor:
+            db_data_version = int(row[0])
+        cursor.execute('SELECT field_value FROM metadata WHERE field_name = \'schema_version\'')
+        for row in cursor:
+            db_schema_version = int(row[0])
+        cursor.close()
+        db.close()
+    except KeyboardInterrupt:
+        raise
+    except:
+        return True
+    return data_version != db_data_version or GAMEDATA_SCHEMA_VERSION != db_schema_version
+
+
+def update_db():
+
+    if os.path.isfile(DB_PATH):
+        os.remove(DB_PATH)
 
-    # Import eos.config first and change it
-    import eos.config
-    eos.config.gamedata_connectionstring = db
-    eos.config.debug = False
-
-    # Now thats done, we can import the eos modules using the config
     import eos.db
     import eos.gamedata
 
@@ -274,7 +304,7 @@ def compareAttrs(attrs1, attrs2):
 
     # Dump all data to memory so we can easely cross check ignored rows
     for jsonName, (minerName, cls) in tables.items():
-        with open(os.path.join(jsonPath, minerName, '{}.json'.format(jsonName)), encoding='utf-8') as f:
+        with open(os.path.join(JSON_DIR, minerName, '{}.json'.format(jsonName)), encoding='utf-8') as f:
             tableData = json.load(f)
         if jsonName in rowsInValues:
             newTableData = []
@@ -373,7 +403,7 @@ def isIgnored(file, row):
                 eos.db.gamedata_session.add(instance)
 
     # quick and dirty hack to get this data in
-    with open(os.path.join(jsonPath, 'fsd_binary', 'dynamicitemattributes.json'), encoding='utf-8') as f:
+    with open(os.path.join(JSON_DIR, 'fsd_binary', 'dynamicitemattributes.json'), encoding='utf-8') as f:
         bulkdata = json.load(f)
         for mutaID, data in bulkdata.items():
             muta = eos.gamedata.DynamicItem()
@@ -402,10 +432,8 @@ def isIgnored(file, row):
     # pyfa, we can do it here as a post-processing step
     eos.db.gamedata_engine.execute('UPDATE dgmtypeattribs SET value = 4.0 WHERE attributeID = ?', (1367,))
 
-    eos.db.gamedata_engine.execute('UPDATE invtypes  SET published = 0 WHERE typeName LIKE \'%abyssal%\'')
+    eos.db.gamedata_engine.execute('UPDATE invtypes SET published = 0 WHERE typeName LIKE \'%abyssal%\'')
 
-    # fix for #1722 until CCP gets their shit together
-    eos.db.gamedata_engine.execute('UPDATE invtypes SET typeName = \'Small Abyssal Energy Nosferatu\' WHERE typeID = ? AND typeName = ?', (48419, ''))
 
     print()
     for x in CATEGORIES_TO_REMOVE:
@@ -418,11 +446,6 @@ def isIgnored(file, row):
 
     print('done')
 
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='This scripts dumps effects from an sqlite cache dump to mongo')
-    parser.add_argument('-d', '--db', required=True, type=str, help='The sqlalchemy connectionstring, example: sqlite:///c:/tq.db')
-    parser.add_argument('-j', '--json', required=True, type=str, help='The path to the json dump')
-    args = parser.parse_args()
-
-    main(args.db, args.json)
 
+if __name__ == '__main__':
+    update_db()

eve.db

@@ -26,6 +26,7 @@
 
 import config
 from service.prereqsCheck import PreCheckException, PreCheckMessage, version_block, version_precheck
+from db_update import db_needs_update, update_db
 
 
 # ascii_text = '''
@@ -116,6 +117,10 @@ def _process_args(self, largs, rargs, values):
         else:
             pyfalog.info("Running in a thawed state.")
 
+        if db_needs_update() is True:
+            pyfalog.info("Gamedata needs an update")
+            update_db()
+
         # Lets get to the good stuff, shall we?
         import eos.db
         import eos.events  # todo: move this to eos initialization?