Only repopulate Price tables when .prices changes

This saves us doing a full cache rebuild every time which improves performance
eyeonus · Nov 22, 2014 · bbfb091 · bbfb091
1 parent 6ff847b
commit bbfb091
Show file tree

Hide file tree

Showing 4 changed files with 90 additions and 56 deletions.
diff --git a/CHANGES.txt b/CHANGES.txt
@@ -9,6 +9,7 @@ v6.0.4 Nov 21 2014 (Beta 3.9.1)
 . (kfsone) Update GUI now selects the first column of the first line on startup
 . (kfsone) Update GUI now supports up/down key inputs and has a default size
 . (kfsone) Issue#57 Fixed extra cache rebuilds after doing an 'update'
+. (kfsone) Changing .prices won't force a full rebuild of the cache
 
 v6.0.3 Nov 21 2014
 . (kfsone) Added a GUI to the "update" sub-command, it's experimental:

diff --git a/cache.py b/cache.py
@@ -155,6 +155,16 @@ def __init__(self, fromFile, lineNo, itemName):
         error = "Unrecognized item name, '{}'.".format(itemName)
         super().__init__(fromFile, lineNo, error)
 
+class UnknownCategoryError(BuildCacheBaseException):
+    """
+        Raised in the case of a categrory name that we don't know.
+        Attributes:
+            categoryName   Key we tried to look up.
+    """
+    def __init__(self, fromFile, lineNo, categoryName):
+        error = "Unrecognized category name, '{}'.".format(categoryName)
+        super().__init__(fromFile, lineNo, error)
+
 
 class DuplicateKeyError(BuildCacheBaseException):
     """
@@ -423,6 +433,12 @@ def changeStation(matches):
         processedStations[stationID] = lineNo
         processedItems = {}
 
+        # Clear old entries for this station.
+        db.execute(
+            "DELETE FROM StationItem WHERE station_id = ?",
+                [stationID]
+        )
+
 
     def changeCategory(matches):
         nonlocal uiOrder, categoryID
@@ -445,7 +461,7 @@ def changeCategory(matches):
             categoryID = categoriesByName[categoryName]
             tdenv.DEBUG1("Renamed: {}", categoryName)
         except KeyError:
-            raise UnknownCategoryError(priceFile, lineNo, facility)
+            raise UnknownCategoryError(priceFile, lineNo, categoryName)
 
 
     def processItemLine(matches):
@@ -553,16 +569,9 @@ def processItemLine(matches):
 
 ######################################################################
 
-def processPricesFile(tdenv, db, pricesPath, stationID=None, defaultZero=False):
+def processPricesFile(tdenv, db, pricesPath, defaultZero=False):
     tdenv.DEBUG0("Processing Prices file '{}'", pricesPath)
 
-    if stationID:
-        tdenv.DEBUG0("Deleting stale entries for {}", stationID)
-        db.execute(
-            "DELETE FROM StationItem WHERE station_id = ?",
-                [stationID]
-        )
-
     with pricesPath.open() as pricesFile:
         items, buys, sells = [], [], []
         for price in genSQLFromPriceLines(tdenv, pricesFile, db, defaultZero):
@@ -814,3 +823,39 @@ def buildCache(tdenv, dbPath, sqlPath, pricesPath, importTables, defaultZero=Fal
 
     tdenv.DEBUG0("Finished")
 
+
+######################################################################
+
+def importDataFromFile(tdenv, tdb, path, reset=False):
+    """
+        Import price data from a file on a per-station basis,
+        that is when a new station is encountered, delete any
+        existing records for that station in the database.
+    """
+
+    if reset:
+        tdenv.DEBUG0("Resetting price data")
+        tdb.cur.execute("DELETE FROM StationItem")
+
+    tdenv.DEBUG0("Importing data from {}".format(str(path)))
+    processPricesFile(tdenv,
+            db=tdb.getDB(),
+            pricesPath=path,
+            defaultZero=tdenv.forceNa,
+            )
+
+    # If everything worked, we may need to re-build the prices file.
+    if path != tdb.pricesPath:
+        import prices
+        tdenv.DEBUG0("Update complete, regenerating .prices file")
+        with tdb.pricesPath.open("w") as pricesFile:
+            prices.dumpPrices(
+                    tdb.dbURI,
+                    prices.Element.full,
+                    file=pricesFile,
+                    debug=tdenv.debug)
+
+    # Update the DB file so we don't regenerate it.
+    os.utime(tdb.dbURI)
+
+
diff --git a/commands/update_cmd.py b/commands/update_cmd.py
@@ -138,25 +138,6 @@ def getEditorPaths(cmdenv, editorName, envVar, windowsFolders, winExe, nixExe):
             )
 
 
-def importDataFromFile(cmdenv, tdb, path, stationID, dbFilename):
-    cmdenv.DEBUG0("Importing data from {}".format(str(path)))
-    cache.processPricesFile(cmdenv,
-                     db=tdb.getDB(),
-                     pricesPath=path,
-                     stationID=stationID,
-                     defaultZero=cmdenv.forceNa
-            )
-
-    # If everything worked, we need to re-build the prices file.
-    cmdenv.DEBUG0("Update complete, regenerating .prices file")
-
-    with tdb.pricesPath.open("w") as pricesFile:
-        prices.dumpPrices(dbFilename, prices.Element.full, file=pricesFile, debug=cmdenv.debug)
-
-    # Update the DB file so we don't regenerate it.
-    os.utime(dbFilename)
-
-
 def editUpdate(tdb, cmdenv, stationID):
     """
         Dump the price data for a specific station to a file and
@@ -264,7 +245,7 @@ def editUpdate(tdb, cmdenv, stationID):
                         "Suit you, sir! Oh!"
                     ])))
         else:
-            importDataFromFile(cmdenv, tdb, tmpPath, stationID, dbFilename)
+            cache.importDataFromFile(cmdenv, tdb, tmpPath)
 
         tmpPath.unlink()
         tmpPath = None

diff --git a/tradedb.py b/tradedb.py
@@ -22,6 +22,7 @@
 
 from tradeexcept import TradeException
 from tradeenv import TradeEnv
+import cache
 
 import locale
 locale.setlocale(locale.LC_ALL, '')
@@ -432,38 +433,44 @@ def reloadCache(self):
         """
 
         if self.dbPath.exists():
-            # We're looking to see if the .sql file or .prices file
-            # was modified or created more recently than the last time
-            # we created/modified the db file.
-            def getMostRecentTimestamp(altPath):
-                try:
-                    stat = altPath.stat()
-                    return max(stat.st_mtime, stat.st_ctime)
-                except FileNotFoundError:
-                    return 0
-
-            dbFileCreatedTimestamp = getMostRecentTimestamp(self.dbPath)
-            sqlTimestamp = getMostRecentTimestamp(self.sqlPath)
-            pricesTimestamp = getMostRecentTimestamp(self.pricesPath)
-
-            # rebuild if the sql or prices file is more recent than the db file
-            if max(sqlTimestamp, pricesTimestamp) < dbFileCreatedTimestamp:
-                # sql and prices file are older than the db, db may be upto date,
-                # check if any of the table files have changed.
-                changedFiles = [ fileName for (fileName, _) in self.importTables if getMostRecentTimestamp(Path(fileName)) > dbFileCreatedTimestamp ]
-                if not changedFiles:
+            dbFileStamp = self.dbPath.stat().st_mtime
+
+            paths = [ self.sqlPath ]
+            paths += [ Path(f) for (f, _) in self.importTables ]
+
+            changedPaths = [
+                    [path, path.stat().st_mtime]
+                        for path in paths
+                        if path.exists() and
+                            path.stat().st_mtime > dbFileStamp
+                    ]
+
+            if not changedPaths:
+                # Do we need to reload the .prices file?
+                if not self.pricesPath.exists():
+                    self.tdenv.DEBUG1("No .prices file to load")
+                    return
+
+                pricesStamp = self.pricesPath.stat().st_mtime
+                if pricesStamp <= dbFileStamp:
                     self.tdenv.DEBUG1("DB Cache is up to date.")
                     return
-                self.tdenv.DEBUG0("Rebuilding DB Cache because of modified {}",
-                                        ', '.join(changedFiles))
-            else:
-                self.tdenv.DEBUG0("Rebuilding DB Cache [db:{}, sql:{}, prices:{}]",
-                                        dbFileCreatedTimestamp, sqlTimestamp, pricesTimestamp)
+
+                self.tdenv.DEBUG0(".prices has changed: re-importing")
+                cache.importDataFromFile(self.tdenv, self, self.pricesPath)
+                return
+
+            self.tdenv.DEBUG0("Rebuilding DB Cache [{}]", str(changedPaths))
         else:
             self.tdenv.DEBUG0("Building DB Cache")
 
-        import cache
-        cache.buildCache(self.tdenv, dbPath=self.dbPath, sqlPath=self.sqlPath, pricesPath=self.pricesPath, importTables=self.importTables)
+        cache.buildCache(
+                self.tdenv,
+                dbPath=self.dbPath,
+                sqlPath=self.sqlPath,
+                pricesPath=self.pricesPath,
+                importTables=self.importTables
+                )
 
 
     ############################################################