From fc35b356d26f3aedf642ae3c467128f319f1c898 Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Thu, 4 Dec 2014 19:41:47 -0800 Subject: [PATCH 1/6] We know item names are going to be unique in future --- cache.py | 48 +++++++----------------------------------------- 1 file changed, 7 insertions(+), 41 deletions(-) diff --git a/cache.py b/cache.py index 16119154..83596bbd 100644 --- a/cache.py +++ b/cache.py @@ -263,44 +263,13 @@ def getCategoriesByNameIndex(cur): return { name: ID for (ID, name) in cur } -def testItemNamesUniqueAcrossCategories(cur): +def getItemByNameIndex(cur): """ - Return True if no single item name appears twice. - In previous betas we had some items which appeared - in more than one category. - """ - cur.execute(""" - SELECT COUNT(*) - FROM ( - SELECT name - FROM Item - GROUP BY name - HAVING COUNT(*) > 1 - ) - """) - nonUniqueItemCount = cur.fetchone()[0] - return (nonUniqueItemCount == 0) - - -def getItemByNameIndex(cur, itemNamesAreUnique): - """ - Generate item name index. If item names are not + Generate item name index. unique, prefix the name with the category id. """ - if itemNamesAreUnique: - cur.execute("SELECT item_id, name FROM item") - return { - name: itemID - for (itemID, name) - in cur - } - else: - cur.execute("SELECT category_id, item_id, name FROM item") - return { - "{}:{}".format(catID, name): itemID - for (catID, itemID, name) - in cur - } + cur.execute("SELECT item_id, name FROM item") + return { name: itemID for (itemID, name) in cur } def genSQLFromPriceLines(tdenv, priceFile, db, defaultZero): @@ -316,8 +285,7 @@ def genSQLFromPriceLines(tdenv, priceFile, db, defaultZero): systemByName = getSystemByNameIndex(cur) categoriesByName = getCategoriesByNameIndex(cur) - itemNamesAreUnique = testItemNamesUniqueAcrossCategories(cur) - itemByName = getItemByNameIndex(cur, itemNamesAreUnique) + itemByName = getItemByNameIndex(cur) defaultUnits = -1 if not defaultZero else 0 defaultLevel = -1 if not defaultZero else 0 @@ -331,7 +299,6 @@ def genSQLFromPriceLines(tdenv, priceFile, db, defaultZero): itemPrefix = "" DELETED = corrections.DELETED - def changeStation(matches): nonlocal categoryID, uiOrder, facility, stationID nonlocal processedStations, processedItems @@ -461,9 +428,8 @@ def processItemLine(matches): modified = None # Use CURRENT_FILESTAMP # Look up the item ID. - itemPrefix = "" if itemNamesAreUnique else "{}:".format(categoryID) try: - itemID = itemByName[itemPrefix + itemName] + itemID = itemByName[itemName] except KeyError: itemID = -1 if itemID < 0: @@ -473,7 +439,7 @@ def processItemLine(matches): tdenv.DEBUG1("DELETED {}", oldName) return try: - itemID = itemByName[itemPrefix + itemName] + itemID = itemByName[itemName] tdenv.DEBUG1("Renamed {} -> {}", oldName, itemName) except KeyError: ex = UnknownItemError(priceFile, lineNo, itemName) From ebba4d2b3cdf85af652b4cc59e1aecf53728e764 Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Thu, 4 Dec 2014 21:35:26 -0800 Subject: [PATCH 2/6] Optimization pass of dump prices Reduced time to generate .prices file on my macbook air from 3.xs to 1.2s; part of this includes always listing supply now. People who don't want to bother with supply can just leave it empty. This in turn will feed into not having to worry about whether or not supply values are supplied in the .prices parser making it much faster --- prices.py | 201 +++++++++++++++++++++++++++++------------------------- 1 file changed, 107 insertions(+), 94 deletions(-) diff --git a/prices.py b/prices.py index 7c87ce03..3dd6a0ae 100644 --- a/prices.py +++ b/prices.py @@ -24,14 +24,24 @@ class Element(object): ###################################################################### # Main -def dumpPrices(dbFilename, elementMask, stationID=None, file=None, defaultZero=False, debug=0): - """ Generate a 'prices' list for the given list of stations using data from the DB. """ +def dumpPrices( + dbPath, # Path() or str + elementMask, # which columns to output + stationID=None, # limits to one station + file=None, # file handle to write to + defaultZero=False, + debug=0 + ): + """ + Generate a prices list using data from the DB. + If stationID is not none, only the specified station is dumped. + If file is not none, outputs to the given file handle. + """ - withSupply = (elementMask & Element.supply) withTimes = (elementMask & Element.timestamp) getBlanks = (elementMask & Element.blanks) - conn = sqlite3.connect(str(dbFilename)) # so we can handle a Path object too + conn = sqlite3.connect(str(dbPath)) conn.execute("PRAGMA foreign_keys=ON") cur = conn.cursor() @@ -43,7 +53,7 @@ def dumpPrices(dbFilename, elementMask, stationID=None, file=None, defaultZero=F } categories = { ID: name for (ID, name) in cur.execute("SELECT category_id, name FROM Category") } items = { - ID: [ name, categories[catID] ] + ID: [ name, catID, categories[catID] ] for (ID, name, catID) in cur.execute("SELECT item_id, name, category_id FROM Item") } @@ -118,105 +128,108 @@ def dumpPrices(dbFilename, elementMask, stationID=None, file=None, defaultZero=F if not file: file = sys.stdout if stationID: - file.write("# TradeDangerous prices for {}\n".format(stations[stationID])) + stationSet = str(stations[stationID]) else: - file.write("# TradeDangerous prices for ALL Systems/Stations\n") - file.write("\n") - - file.write("# REMOVE ITEMS THAT DON'T APPEAR IN THE UI\n") - file.write("# ORDER IS REMEMBERED: Move items around within categories to match the game UI\n") - file.write("\n") - - file.write("# File syntax:\n") - file.write("# [ [] ]\n") - file.write("# '?' or 'unk' indicates unknown values (don't care),\n") - file.write("# '-' or 'n/a' indicates 'not available' item,\n") - file.write("# Level can be '?', 'L', 'M' or 'H'\n") - file.write("# If you omit the timestamp, the current time will be used when the file is loaded.\n") - - file.write("\n") - - levelDescriptions = { - -1: "?", - 0: "0", - 1: "L", - 2: "M", - 3: "H" - } - def itemQtyAndLevel(quantity, level): - if defaultZero and quantity == -1 and level == -1: - quantity, level = 0, 0 - if quantity < 0 and level < 0: - return "?" - if quantity == 0 and level == 0: - return "-" - # Quantity of -1 indicates 'unknown' - quantityDesc = '?' if quantity < 0 else str(quantity) - # try to use a descriptive for the level - try: - levelDesc = levelDescriptions[int(level)] - except (KeyError, ValueError): - levelDesc = str(level) - return "{}{}".format(quantityDesc, levelDesc) - + stationSet = "ALL Systems/Stations" + + file.write( + "# TradeDangerous prices for {}\n" + "\n" + "# REMOVE ITEMS THAT DON'T APPEAR IN THE UI\n" + "# ORDER IS REMEMBERED: Move items around within categories " + "to match the game UI\n" + "\n" + "# File syntax:\n" + "# [ []]\n" + "# Use '?' for demand/stock when you don't know/care,\n" + "# Use '-' for demand/stock to indicate unavailable,\n" + "# Otherwise use a number followed by L, M or H, e.g.\n" + "# 1L, 23M or 30000H\n" + "# If you omit the timestamp, the current time will be used when " + "the file is loaded.\n" + "\n".format( + stationSet + )) + + levelDesc = "?0LMH" maxCrWidth = 7 - levelWidth = 8 - - file.write("# {:<{width}} {:>{crwidth}} {:>{crwidth}}".format("Item Name", "Sell Cr", "Buy Cr", width=longestNameLen, crwidth=maxCrWidth)) - if withSupply: - file.write(" {:>{lvlwidth}} {:>{lvlwidth}}".format("Demand", "Stock", lvlwidth=levelWidth)) + levelWidth = 9 + + outFmt = ( + " {{:<{width}}}" + " {{:>{crwidth}}}" + " {{:>{crwidth}}}" + " {{:>{lvlwidth}}}" + " {{:>{lvlwidth}}}".format( + width=longestNameLen, + crwidth=maxCrWidth, + lvlwidth=levelWidth, + ) + ) if withTimes: - file.write(" {}".format("Timestamp")) - file.write("\n\n") + outFmt += " {}" + outFmt += "\n" + output = outFmt.format( + "Item Name", + "SellCr", "BuyCr", + "Demand", "Stock", + "Timestamp", + ) + file.write('#' + output[1:]) - naIQL = itemQtyAndLevel(0, 0) - unkIQL = itemQtyAndLevel(-2, -2) - defIQL = itemQtyAndLevel(-1, -1) + naIQL = "-" + unkIQL = "?" + defIQL = "?" if not defaultZero else "-" + output = "" for (stnID, itemID, fromStn, toStn, modified, demand, demandLevel, stock, stockLevel) in cur: modified = modified or now station, system = stations[stnID] - if system is not lastSys: - if lastStn: file.write("\n\n") - lastStn, lastCat = None, None - lastSys = system - if station is not lastStn: - if lastStn: file.write("\n") + item, catID, category = items[itemID] + if stnID != lastStn: + file.write(output) + output = "\n\n@ {}/{}\n".format(system.upper(), station) + lastStn = stnID lastCat = None - file.write("@ {}/{}\n".format(system.upper(), station)) - lastStn = station - - item, category = items[itemID] - if category is not lastCat: - file.write(" + {}\n".format(category)) - lastCat = category - - file.write(" {:<{width}} {:{crwidth}d} {:{crwidth}d}".format( - item, fromStn, toStn, - width=longestNameLen, crwidth=maxCrWidth - )) - if withSupply: - # Is this item on sale? - if toStn > 0: - # Zero demand-price gets default demand, which will - # be either unknown or zero depending on -0. - # If there is a price, always default to unknown - # because it can be sold here but the demand is just - # not useful as data. - demandStr = defIQL if fromStn <= 0 else unkIQL - stockStr = itemQtyAndLevel(stock, stockLevel) - else: - demandStr = itemQtyAndLevel(demand, demandLevel) - stockStr = naIQL - file.write(" {:>{lvlwidth}} {:>{lvlwidth}}".format( - demandStr, - stockStr, - lvlwidth=levelWidth, - )) - if withTimes and modified: - file.write(" {}".format(modified)) - file.write("\n") + if catID is not lastCat: + output += " + {}\n".format(category) + lastCat = catID + + # Is this item on sale? + if toStn > 0: + # Zero demand-price gets default demand, which will + # be either unknown or zero depending on -0. + # If there is a price, always default to unknown + # because it can be sold here but the demand is just + # not useful as data. + demandStr = defIQL if fromStn <= 0 else unkIQL + if stockLevel == 0: + stockStr = naIQL + elif stockLevel < 0 and stock <= 0: + stockStr = defIQL + else: + units = "?" if stock < 0 else str(stock) + level = levelDesc[stockLevel + 1] + stockStr = units + level + else: + if demandLevel == 0: + demandStr = naIQL + elif demandLevel < 0 and demand <= 0: + demandStr = defIQL + else: + units = "?" if demand < 0 else str(demand) + level = levelDesc[demandLevel + 1] + demandStr = units + level + stockStr = naIQL + output += outFmt.format( + item, + fromStn, toStn, + demandStr, stockStr, + modified + ) + + file.write(output) if __name__ == "__main__": from tradedb import TradeDB From 97029a8cdc5f692cc5cb1c4ef266d291c1360c97 Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Thu, 4 Dec 2014 21:36:42 -0800 Subject: [PATCH 3/6] Herptimization Ok - so we were spending ~20ms processing avoids on startup when no avoids were specified. Reduced that to .2ms. --- commands/commandenv.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/commands/commandenv.py b/commands/commandenv.py index fd73c92b..3fe937d2 100644 --- a/commands/commandenv.py +++ b/commands/commandenv.py @@ -150,14 +150,10 @@ def checkAvoids(self): avoidItems = self.avoidItems = [] avoidPlaces = self.avoidPlaces = [] - - try: - avoidances = self.avoid - if not avoidances: - raise AttributeError("Fake") - except AttributeError: - self.avoidPlaces = [] + avoidances = self.avoid + if not self.avoid: return + avoidances = self.avoid tdb = self.tdb From 6fb8f084f3dc6c7011febe1c36db2f2894c8edb5 Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Thu, 4 Dec 2014 21:37:01 -0800 Subject: [PATCH 4/6] "--supply" update switch is now deprecated --- README.txt | 58 +++++++++++++++++++++++++++--------------- commands/update_cmd.py | 5 +++- 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/README.txt b/README.txt index 0fdc6056..d71d0618 100644 --- a/README.txt +++ b/README.txt @@ -322,10 +322,6 @@ UPDATE sub-command: Saves the prices in a human-readable format and loads that into an editor. Make changes and save to update the database. - --supply - -S - Exposes the "demand" and "stock" columns. - --timestamps -T Exposes the "timestamp" column. @@ -656,37 +652,60 @@ is designed to closely resemble what we see in the market screens in-game. To edit the data for a single station, use the "update" sub-command, e.g. - trade.py update --notepad Aulin + trade.py update --notepad beagle2 This will open notepad with the data for Aulin, which will look something like: - @ AULIN/Aulin Enterprise + @ I BOOTIS/Beagle 2 Landing + Chemicals - Explosives 50 0 - Hydrogen Fuels 19 0 - Mineral Oil 100 0 - Pesticides 21 0 + Explosives 50 0 ? - + Hydrogen Fuels 19 0 ? - + Mineral Oil 100 0 ? - + Pesticides 21 0 ? - + Consumer Items - Clothing 300 0 - Consumer Tech 1112 1111 + Clothing 300 0 ? - + Consumer Tech 1112 1111 ? 30L "@" lines specify a system/station. "+" lines specify a category. The remaining lines are items. - Explosives 50 0 + Explosives 50 0 ? - These fields are: (how much the station pays) (how much the station charges) + ('?' means "we don't care") + ('-' means "not available") So you can see the only item this station is selling is Consumer Tech, which -the station wants 1111 credits for. - -TradeDangerous can also leverage the "DEMAND" and "STOCK" values from the -market UI. To expose those when using "trade.py update", use the "--all" -option. For more details of the extended format, see the Prices wiki page: +the station wants 1111 credits for. The demand wasn't recorded (we generally +aren't interested in demand levels since E:D doesn't seem to use them) and +the items wasn't available for purchase *from* the station. + +TD will use the 'stock' values to limit how many units it can buy. For +example, if you have money to buy 30 units from a station but the .prices +data says only 10 are available, TD only tell you to buy 10 units and it +will fill the rest of your hold up with other stuff. + +Demand and Stock both take a "supply level" value which is either "?", "-" +or the number of units followed by the level: L for Low, M for Medium, +H for High or ? for "don't care". + + ? + - + 1L + 23M + 3402H + 444? + +Best Practice: + + - Leave demand as ?, neither E:D or TD use it, + - Stock quantities over 10k are usuall irrelevant, leave them as ?, + +For more details of the .prices format, see the wiki page: https://bitbucket.org/kfsone/tradedangerous/wiki/Price%20Data NOTE: The order items are listed within their category is saved between edits, @@ -695,9 +714,6 @@ will show that way when you edit this station again. See "trade.py update -h" for more help with the update command. -Note: My personal editor of choice is "Sublime Text", which is why there is -a command line option (--sublime or just --subl) for invoking it. - ============================================================================== == That's nice, but I'm a programmer and I want to ... diff --git a/commands/update_cmd.py b/commands/update_cmd.py index 1da063d4..8a102982 100644 --- a/commands/update_cmd.py +++ b/commands/update_cmd.py @@ -26,7 +26,7 @@ ] switches = [ ParseArgument('--supply', '-S', - help='[Text editing] Includes demand and stock (supply) values in the update.', + help='[DEPRECATED] Includes demand and stock (supply) values in the update.', action='store_true', default=False, ), @@ -381,6 +381,9 @@ def run(results, cmdenv, tdb): "flags.\n" ) + if not cmdenv.quiet and cmdenv.supply: + print("NOTE: '--supply' (-S) is deprecated.") + # User specified one of the options to use an editor. editUpdate(tdb, cmdenv, cmdenv.startStation.ID) From 87ca598fa6f23a3dea90ea678ba3fc13b3cb500a Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Thu, 4 Dec 2014 21:53:08 -0800 Subject: [PATCH 5/6] Only try system-specific corrections for station names --- corrections.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/corrections.py b/corrections.py index a3b5d23b..849b3eaf 100644 --- a/corrections.py +++ b/corrections.py @@ -42,10 +42,6 @@ def correctSystem(oldName): def correctStation(systemName, oldName): try: return stations[systemName.upper() + "/" + oldName.upper()] - except KeyError: - pass - try: - return stations[oldName.upper()] except KeyError: return oldName From 8afc5edb07e95a547ce6584fe30fb8b41ad43b30 Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Thu, 4 Dec 2014 22:06:26 -0800 Subject: [PATCH 6/6] Performance pass for buildcache/import Improved performance of .prices parsing to shave several seconds off the time it takes to build the cache. --- CHANGES.txt | 5 + cache.py | 231 ++++++++++++++++++------------------- commands/buildcache_cmd.py | 10 -- 3 files changed, 115 insertions(+), 131 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1e948035..2416ecf3 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,6 +2,11 @@ TradeDangerous, Copyright (C) Oliver "kfsone" Smith, July 2014 ============================================================================== +v6.1.5 Dec 04 2014 +. (kfsone) Startup optimizations, +. (kfsone) "update" command now always exposes supply and demand, +. (kfsone) "--supply" (-S) is now deprecated for "update" command, + v6.1.4 Dec 01 2014 . (kfsone) "updated.prices" will now contain line removals . (kfsone) 155 new systems from RedWizard diff --git a/cache.py b/cache.py index 83596bbd..4f0126d2 100644 --- a/cache.py +++ b/cache.py @@ -36,14 +36,11 @@ # - use named captures (?P ...) # - include comments -## Find the non-comment part of a string -noCommentRe = re.compile(r'^\s*(?P(?:[^\\#]|\\.)*)\s*(#|$)') - ## Match the '@ SYSTEM/Station' line -systemStationRe = re.compile(r'^\@\s*(.*)\s*/\s*(.*)') +systemStationRe = re.compile(r'^\@\s*(.*)/(.*)') ## Match the '+ Category' line -categoryRe = re.compile(r'^\+\s*(.*?)\s*$') +categoryRe = re.compile(r'^\+\s*(.*)') ## Price Line matching @@ -53,10 +50,10 @@ (?P .*?) \s+ # price station is buying the item for - (?P \d+) + (?P \d+) \s+ # price station is selling item for - (?P \d+) + (?P \d+) """ # time formats per https://www.sqlite.org/lang_datefunc.html @@ -200,28 +197,22 @@ def __init__(self, fromFile, lineNo, category, problem, value): # Helpers def parseSupply(pricesFile, lineNo, category, reading): - if len(reading) == 1: - if reading == "?": - return (-1, -1) - elif reading == "-" or reading == "0": - return (0, 0) - else: - units, level = reading[0:-1], reading[-1] - levelNo = "??LMH".find(level.upper()) -1 - if levelNo < -1: - raise SupplyError( - pricesFile, lineNo, category, reading, - "Unrecognized level suffix '{}', " - "expected one of 'L', 'M', 'H' or '?'".format( - level - )) - try: - unitsNo = int(units) - if unitsNo <= 0: - raise ValueError("unsigned unit count") - return (unitsNo, levelNo) - except ValueError: - pass + units, level = reading[0:-1], reading[-1] + levelNo = "??LMH".find(level.upper()) -1 + if levelNo < -1: + raise SupplyError( + pricesFile, lineNo, category, reading, + "Unrecognized level suffix '{}', " + "expected one of 'L', 'M', 'H' or '?'".format( + level + )) + try: + unitsNo = int(units) + if unitsNo <= 0: + raise ValueError("unsigned unit count") + return unitsNo, levelNo + except ValueError: + pass raise SupplyError( pricesFile, lineNo, category, reading, @@ -232,15 +223,6 @@ def parseSupply(pricesFile, lineNo, category, reading): )) -class PriceEntry(namedtuple('PriceEntry', [ - 'stationID', 'itemID', - 'uiOrder', 'modified', - 'sellTo', 'demand', 'demandLevel', - 'buyFrom', 'stock', 'stockLevel', - ])): - pass - - ###################################################################### # Code ###################################################################### @@ -272,7 +254,7 @@ def getItemByNameIndex(cur): return { name: itemID for (itemID, name) in cur } -def genSQLFromPriceLines(tdenv, priceFile, db, defaultZero): +def processPrices(tdenv, priceFile, db, defaultZero): """ Yields SQL for populating the database with prices by reading the file handle for price lines. @@ -298,6 +280,7 @@ def genSQLFromPriceLines(tdenv, priceFile, db, defaultZero): processedItems = {} itemPrefix = "" DELETED = corrections.DELETED + items, buys, sells = [], [], [] def changeStation(matches): nonlocal categoryID, uiOrder, facility, stationID @@ -318,46 +301,40 @@ def changeStation(matches): stationID = -1 if stationID < 0: - systemName = corrections.correctSystem(systemName).upper() - stationName = corrections.correctStation(systemName, stationName).upper() - if systemName == DELETED or stationName == DELETED: - tdenv.DEBUG1("DELETED: {}", facility) - stationID = DELETED - return + try: + correctName = corrections.systems[systemName] + if correction == DELETED: + tdenv.DEBUG1("DELETED: {}", systemName) + stationID = correction + return + systemName = correction.upper() + except KeyError: + pass + try: + key = systemName + "/" + stationName.upper() + correction = corrections.stations[key] + if correction == DELETED: + tdenv.DEBUG1("DELETED: {}", key) + stationID = correction + return + stationName = correction.upper() + except KeyError: + pass facility = systemName + '/' + stationName try: stationID = systemByName[facility] tdenv.DEBUG0("Renamed: {}", facility) except KeyError: - if tdenv.corrections: - ### HACK: HERE BE DRAEGONS - # Stations we didn't have a name for are named {STAR}-STATION, - # this is a catch-all for the case where we have a .prices - # that provides the actual name for the station but we don't - # yet have a corrections.py/Station.csv entry for it. - # Run with --corrections to generate a list of corrections.py - # additions. - altStationName = systemName + "-STATION" - altFacility = systemName + '/' + altStationName - try: - stationID = systemByName[altFacility] - except KeyError: - stationID = -1 - if stationID >= 0: - facility = altFacility - if altStationName in corrections.stations: - raise Exception(altStationName + " conflicts") - print(" \"{}\": \"{}\",".format( - facility, stationNameIn, - ), file=sys.stderr) - if stationID < 0 : - ex = UnknownStationError(priceFile, lineNo, facility) - if not tdenv.ignoreUnknown: - raise ex - stationID = DELETED - if not tdenv.quiet: - print(ex) - return + stationID = -1 + + if stationID < 0 : + ex = UnknownStationError(priceFile, lineNo, facility) + if not tdenv.ignoreUnknown: + raise ex + stationID = DELETED + if not tdenv.quiet: + print(ex) + return # Check for duplicates if stationID in processedStations: @@ -379,7 +356,6 @@ def changeStation(matches): def changeCategory(matches): nonlocal uiOrder, categoryID, categoryName - categoryID = -1 categoryName, uiOrder = matches.group(1), 0 tdenv.DEBUG1("NEW CATEGORY: {}", categoryName) @@ -409,23 +385,8 @@ def changeCategory(matches): def processItemLine(matches): nonlocal uiOrder, processedItems + nonlocal items, buys, sells itemName, modified = matches.group('item', 'time') - sellTo = int(matches.group('sell_to')) - buyFrom = int(matches.group('buy_from')) - demandString, stockString = matches.group('demand', 'stock') - if demandString and stockString: - demandUnits, demandLevel = parseSupply( - priceFile, lineNo, 'demand', demandString - ) - stockUnits, stockLevel = parseSupply( - priceFile, lineNo, 'stock', stockString - ) - else: - demandUnits, demandLevel = defaultUnits, defaultLevel - stockUnits, stockLevel = defaultUnits, defaultLevel - - if modified == 'now': - modified = None # Use CURRENT_FILESTAMP # Look up the item ID. try: @@ -457,26 +418,66 @@ def processItemLine(matches): processedItems[itemID] ) + sellTo, buyFrom = matches.group('sell', 'buy') + sellTo, buyFrom = int(sellTo), int(buyFrom) + demandString, stockString = matches.group('demand', 'stock') + if demandString and stockString: + if demandString == "?": + demandUnits, demandLevel = -1, -1 + elif demandString == "-": + demandUnits, demandLevel = 0, 0 + else: + demandUnits, demandLevel = parseSupply( + priceFile, lineNo, 'demand', demandString + ) + if stockString == "?": + stockUnits, stockLevel = -1, -1 + elif stockString == "-": + stockUnits, stockLevel = 0, 0 + else: + stockUnits, stockLevel = parseSupply( + priceFile, lineNo, 'stock', stockString + ) + else: + demandUnits, demandLevel = defaultUnits, defaultLevel + stockUnits, stockLevel = defaultUnits, defaultLevel + + if modified == 'now': + modified = None # Use CURRENT_FILESTAMP + processedItems[itemID] = lineNo uiOrder += 1 - return PriceEntry(stationID, itemID, - uiOrder, modified, - sellTo, demandUnits, demandLevel, - buyFrom, stockUnits, stockLevel - ) + items.append([ stationID, itemID, uiOrder, modified ]) + if sellTo > 0 and demandUnits != 0 and demandLevel != 0: + buys.append([ + stationID, itemID, + sellTo, demandUnits, demandLevel, + modified + ]) + if buyFrom > 0 and stockUnits != 0 and stockLevel != 0: + sells.append([ + stationID, itemID, + buyFrom, stockUnits, stockLevel, + modified + ]) for line in priceFile: lineNo += 1 - if len(line) <= 3 or line.startswith('#'): - continue + commentPos = line.find("#") + if commentPos >= 0: + if commentPos == 0: + continue + line = line[:commentPos] + text = line.strip() - text = noCommentRe.match(line).group('text').strip() - # replace whitespace with single spaces - text = ' '.join(text.split()) # http://stackoverflow.com/questions/2077897 if not text: continue + # replace whitespace with single spaces + if text.find(" "): + text = ' '.join(text.split()) # http://stackoverflow.com/questions/2077897 + ######################################## ### "@ STAR/Station" lines. if text.startswith('@'): @@ -521,9 +522,9 @@ def processItemLine(matches): raise SyntaxError(priceFile, lineNo, "Unrecognized line/syntax", text) - sql = processItemLine(matches) - if sql: - yield sql + processItemLine(matches) + + return items, buys, sells ###################################################################### @@ -534,20 +535,7 @@ def processPricesFile(tdenv, db, pricesPath, defaultZero=False): assert isinstance(pricesPath, Path) with pricesPath.open() as pricesFile: - items, buys, sells = [], [], [] - for price in genSQLFromPriceLines(tdenv, pricesFile, db, defaultZero): - tdenv.DEBUG2(str(price)) - stnID, itemID = price.stationID, price.itemID - uiOrder, modified = price.uiOrder, price.modified - items.append([ stnID, itemID, uiOrder, modified ]) - if uiOrder > 0: - cr, units, level = price.sellTo, price.demand, price.demandLevel - if cr > 0 and units != 0 and level != 0: - buys.append([ stnID, itemID, cr, units, level, modified ]) - cr, units, level = price.buyFrom, price.stock, price.stockLevel - if cr > 0 and units != 0 and level != 0: - sells.append([ stnID, itemID, cr, units, level, modified ]) - + items, buys, sells = processPrices(tdenv, pricesFile, db, defaultZero) if items: db.executemany(""" INSERT INTO StationItem @@ -662,9 +650,10 @@ def processImportFile(tdenv, db, importPath, tableName): tdenv.DEBUG0("SQL-Statement: {}", sql_stmt) # Check if there is a deprecation check for this table. - deprecationFn = getattr(sys.modules[__name__], - "deprecationCheck"+tableName, - None) + if tdenv.debug: + deprecationFn = getattr(sys.modules[__name__], + "deprecationCheck"+tableName, + None) # import the data importCount = 0 diff --git a/commands/buildcache_cmd.py b/commands/buildcache_cmd.py index ec3badc6..87e3bf54 100644 --- a/commands/buildcache_cmd.py +++ b/commands/buildcache_cmd.py @@ -33,16 +33,6 @@ dest='force', help='Overwrite existing file', ), - ParseArgument( - '--corrections', default=False, action='store_true', - help=( - "EXPERT: " - "Allow- and generate corrections for- station names that " - "replace a defaulted station name. Corrections are " - "printed to stderr so you can capture them and add them " - "to your data/corrections.py." - ), - ), ParseArgument( '--ignore-unknown', '-i', default=False, action='store_true',