feat: add maxage option to spansh plugin

By specifying maxage, any station from the source that is older than the
age will be skipped.
So if a full update was done using galaxy_stations.json on 13th May
(which is updated every ~24 hours), doing a new update on 20th of May
with a max_age of 7 will skip anything in the source that wasn't updated
later than 13th May

tradedangerous/cache.py

@@ -632,17 +632,34 @@ def processPricesFile(tdenv, db, pricesPath, pricesFh = None, defaultZero = Fals
     removedItems = len(zeros)
 
     if items:
-        db.executemany("""
-            INSERT OR REPLACE INTO StationItem (
-                station_id, item_id, modified,
-                demand_price, demand_units, demand_level,
-                supply_price, supply_units, supply_level
-            ) VALUES (
-                ?, ?, IFNULL(?, CURRENT_TIMESTAMP),
-                ?, ?, ?,
-                ?, ?, ?
-            )
-        """, items)
+        for item in items:
+            try:
+                db.execute("""
+                    INSERT OR REPLACE INTO StationItem (
+                        station_id, item_id, modified,
+                        demand_price, demand_units, demand_level,
+                        supply_price, supply_units, supply_level
+                    ) VALUES (
+                        ?, ?, IFNULL(?, CURRENT_TIMESTAMP),
+                        ?, ?, ?,
+                        ?, ?, ?
+                    )
+                """, item)
+            except sqlite3.IntegrityError as e:
+                print(e)
+                print(item)
+                raise e
+        # db.executemany("""
+        #     INSERT OR REPLACE INTO StationItem (
+        #         station_id, item_id, modified,
+        #         demand_price, demand_units, demand_level,
+        #         supply_price, supply_units, supply_level
+        #     ) VALUES (
+        #         ?, ?, IFNULL(?, CURRENT_TIMESTAMP),
+        #         ?, ?, ?,
+        #         ?, ?, ?
+        #     )
+        # """, items)
     updatedItems = len(items)
 
     tdenv.DEBUG0("Marking populated stations as having a market")
@@ -654,6 +671,7 @@ def processPricesFile(tdenv, db, pricesPath, pricesFh = None, defaultZero = Fals
              ")"
     )
 
+    tdenv.DEBUG0(f'Committing...')
     db.commit()
 
     changes = " and ".join("{} {}".format(v, k) for k, v in {

tradedangerous/plugins/spansh_plug.py

@@ -55,13 +55,15 @@ class ImportPlugin(plugins.ImportPluginBase):
     pluginOptions = {
         'url':  f'URL to download galaxy data from (defaults to {SOURCE_URL})',
         'file': 'Local filename to import galaxy data from; use "-" to load from stdin',
+        'maxage': 'Skip all entries older than specified age in days, ex.: maxage=1.5',
         'listener': 'For use by TD-listener, prevents updating cache from generated prices file',
     }
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.url = self.getOption('url')
         self.file = self.getOption('file')
+        self.maxage = float(self.getOption('maxage'))
         self.listener = self.getOption('listener')
         assert not (self.url and self.file), 'Provide either url or file, not both'
         if self.file and (self.file != '-'):
@@ -104,6 +106,10 @@ def run(self):
                 station_count = 0
                 commodity_count = 0
                 for station, commodities in stations:
+                    if (datetime.now() - station.modified) > timedelta(days=self.maxage):
+                        if self.tdenv.detail >= 1:
+                            self.print(f'        |  @{system.name.upper()}/{station.name.upper():50s}  |  Skipping station due to age: {datetime.now() - station.modified}, ts: {station.modified}')
+                        continue
                     if (system.name.upper(), station.name.upper()) in seen_stations:
                         fq_station_name = f'@{system.name.upper()}/{station.name}'
                         if self.tdenv.detail >= 1:
@@ -183,15 +189,15 @@ def categorise_commodities(self, commodities):
         return categories
 
     def execute(self, query, *params, **kwparams):
-        attempts = 5
+        # attempts = 5
         cursor = self.tdb.getDB().cursor()
         while True:
             try:
                 return cursor.execute(query, params or kwparams)
             except sqlite3.OperationalError as ex:
-                if not attempts:
-                    raise
-                attempts -= 1
+                # if not attempts:
+                #     raise
+                # attempts -= 1
                 self.print(f'Retrying query: {ex!s}')
                 time.sleep(1)