diff --git a/tradedangerous/cache.py b/tradedangerous/cache.py index 502c4899..5a3fad1b 100644 --- a/tradedangerous/cache.py +++ b/tradedangerous/cache.py @@ -632,17 +632,34 @@ def processPricesFile(tdenv, db, pricesPath, pricesFh = None, defaultZero = Fals removedItems = len(zeros) if items: - db.executemany(""" - INSERT OR REPLACE INTO StationItem ( - station_id, item_id, modified, - demand_price, demand_units, demand_level, - supply_price, supply_units, supply_level - ) VALUES ( - ?, ?, IFNULL(?, CURRENT_TIMESTAMP), - ?, ?, ?, - ?, ?, ? - ) - """, items) + for item in items: + try: + db.execute(""" + INSERT OR REPLACE INTO StationItem ( + station_id, item_id, modified, + demand_price, demand_units, demand_level, + supply_price, supply_units, supply_level + ) VALUES ( + ?, ?, IFNULL(?, CURRENT_TIMESTAMP), + ?, ?, ?, + ?, ?, ? + ) + """, item) + except sqlite3.IntegrityError as e: + print(e) + print(item) + raise e + # db.executemany(""" + # INSERT OR REPLACE INTO StationItem ( + # station_id, item_id, modified, + # demand_price, demand_units, demand_level, + # supply_price, supply_units, supply_level + # ) VALUES ( + # ?, ?, IFNULL(?, CURRENT_TIMESTAMP), + # ?, ?, ?, + # ?, ?, ? + # ) + # """, items) updatedItems = len(items) tdenv.DEBUG0("Marking populated stations as having a market") @@ -654,6 +671,7 @@ def processPricesFile(tdenv, db, pricesPath, pricesFh = None, defaultZero = Fals ")" ) + tdenv.DEBUG0(f'Committing...') db.commit() changes = " and ".join("{} {}".format(v, k) for k, v in { diff --git a/tradedangerous/plugins/spansh_plug.py b/tradedangerous/plugins/spansh_plug.py index 7caecd98..12b8a12b 100644 --- a/tradedangerous/plugins/spansh_plug.py +++ b/tradedangerous/plugins/spansh_plug.py @@ -55,6 +55,7 @@ class ImportPlugin(plugins.ImportPluginBase): pluginOptions = { 'url': f'URL to download galaxy data from (defaults to {SOURCE_URL})', 'file': 'Local filename to import galaxy data from; use "-" to load from stdin', + 'maxage': 'Skip all entries older than specified age in days, ex.: maxage=1.5', 'listener': 'For use by TD-listener, prevents updating cache from generated prices file', } @@ -62,6 +63,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.url = self.getOption('url') self.file = self.getOption('file') + self.maxage = float(self.getOption('maxage')) self.listener = self.getOption('listener') assert not (self.url and self.file), 'Provide either url or file, not both' if self.file and (self.file != '-'): @@ -104,6 +106,10 @@ def run(self): station_count = 0 commodity_count = 0 for station, commodities in stations: + if (datetime.now() - station.modified) > timedelta(days=self.maxage): + if self.tdenv.detail >= 1: + self.print(f' | @{system.name.upper()}/{station.name.upper():50s} | Skipping station due to age: {datetime.now() - station.modified}, ts: {station.modified}') + continue if (system.name.upper(), station.name.upper()) in seen_stations: fq_station_name = f'@{system.name.upper()}/{station.name}' if self.tdenv.detail >= 1: @@ -183,15 +189,15 @@ def categorise_commodities(self, commodities): return categories def execute(self, query, *params, **kwparams): - attempts = 5 + # attempts = 5 cursor = self.tdb.getDB().cursor() while True: try: return cursor.execute(query, params or kwparams) except sqlite3.OperationalError as ex: - if not attempts: - raise - attempts -= 1 + # if not attempts: + # raise + # attempts -= 1 self.print(f'Retrying query: {ex!s}') time.sleep(1)