From 635ca0634c5a026c0dcea6d0498a2ee05b91cde2 Mon Sep 17 00:00:00 2001 From: eyeonus Date: Thu, 27 Sep 2018 14:19:33 -0600 Subject: [PATCH 1/3] Untested attempt to address #25 --- plugins/eddblink_plug.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/plugins/eddblink_plug.py b/plugins/eddblink_plug.py index 3c05379b..cfc48161 100644 --- a/plugins/eddblink_plug.py +++ b/plugins/eddblink_plug.py @@ -630,12 +630,17 @@ def importListings(self, listings_file): progress = 0 total = 1 + if listings_file == LISTINGS: + from_live = 0 + else: + from_live = 1 + def blocks(f, size = 65536): while True: b = f.read(size) if not b: break yield b - + with open(str(self.dataPath / listings_file), "r",encoding = "utf-8",errors = 'ignore') as f: total += (sum(bl.count("\n") for bl in blocks(f))) @@ -643,6 +648,10 @@ def blocks(f, size = 65536): if self.getOption("progbar"): prog = pbar.Progress(total, 50) listings = csv.DictReader(fh) + + cur_station = -1 + station_items = dict() + for listing in listings: if self.getOption("progbar"): prog.increment(1, postfix=lambda value, goal: " " + str(round(value / total * 100)) + "%") @@ -658,8 +667,21 @@ def blocks(f, size = 65536): supply_price = int(listing['buy_price']) supply_units = int(listing['supply']) supply_level = int(listing['supply_bracket']) if listing['supply_bracket'] != '' else -1 - #from_live = 0 if listings_file == LISTINGS else 1 - from_live = 0 + + if station_id != cur_station: + self.execute("BEGIN IMMEDIATE") + for item in station_items: + if not item: + self.execute("DELETE from StationItem WHERE station_id = ? and item_id = ?", (station_id, item)) + del station_items, cur_station, item + cur_station = station_id + station_items = dict() + cursor = self.execute("SELECT item_id from StationItem WHERE station_id = ?", (station_id,)) + for item in cursor: + station_items(item) = False + del cursor, item + + station_items(item_id) = True result = self.execute("SELECT modified FROM StationItem WHERE station_id = ? AND item_id = ?", (station_id, item_id)).fetchone() if result: @@ -707,6 +729,7 @@ def blocks(f, size = 65536): prog.increment(1, postfix=lambda value, goal: " " + str(round(value / total * 100)) + "%") prog.clear() + del from_live self.updated['Listings'] = True tdenv.NOTE("Finished processing market data. End time = {}", datetime.datetime.now()) @@ -740,6 +763,9 @@ def run(self): self.execute("ALTER TABLE Station ADD type_id INTEGER DEFAULT 0 NOT NULL") except sqlite3.OperationalError: pass + except sqlite3.DatabaseError as e: + self.options['clean'] = True + tdenv.NOTE("Cleaning database: ",str(e)) if self.getOption("clean"): # Rebuild the tables from scratch. Must be done on first run of plugin. From 3c0ab291e9babaeb01981e75ec946bcd940c7563 Mon Sep 17 00:00:00 2001 From: eyeonus Date: Sat, 27 Oct 2018 15:52:47 -0600 Subject: [PATCH 2/3] Change env to python3 instead of python3.6 This change allows users with later versions of python3 to run the program without having to make sure they have specifically python3.6 installed. --- trade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trade.py b/trade.py index e869102a..15bb51d7 100755 --- a/trade.py +++ b/trade.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.6 +#!/usr/bin/env python3 # -------------------------------------------------------------------- # Copyright (C) Oliver 'kfsone' Smith 2014 : # Copyright (C) Bernd 'Gazelle' Gollesch 2016, 2017 From 2b617e8884180b361dd787774a7cf25a4ca4525a Mon Sep 17 00:00:00 2001 From: eyeonus Date: Sat, 27 Oct 2018 18:56:37 -0600 Subject: [PATCH 3/3] Fix some syntax errors. --- plugins/eddblink_plug.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/eddblink_plug.py b/plugins/eddblink_plug.py index cfc48161..ef698b9a 100644 --- a/plugins/eddblink_plug.py +++ b/plugins/eddblink_plug.py @@ -669,19 +669,18 @@ def blocks(f, size = 65536): supply_level = int(listing['supply_bracket']) if listing['supply_bracket'] != '' else -1 if station_id != cur_station: - self.execute("BEGIN IMMEDIATE") for item in station_items: if not item: self.execute("DELETE from StationItem WHERE station_id = ? and item_id = ?", (station_id, item)) - del station_items, cur_station, item + del station_items, cur_station cur_station = station_id station_items = dict() cursor = self.execute("SELECT item_id from StationItem WHERE station_id = ?", (station_id,)) for item in cursor: - station_items(item) = False - del cursor, item + station_items[item] = False + del cursor - station_items(item_id) = True + station_items[item_id] = True result = self.execute("SELECT modified FROM StationItem WHERE station_id = ? AND item_id = ?", (station_id, item_id)).fetchone() if result: