Adding support for a cleaner extended .prices format while retaining …

…backwards compatability.

This new format does several things:
1. Makes the timestamp optional in input data and makes it optional as well as accepting 'now' as a value,
2. Removes the words 'demand' and 'stock' from the line,
3. Replaces "-1L-1" with "unk" (unknown),
4. Replaces "0L0" with "n/a" (not available),
5. Replaces 'nnnL1' with "nnn@L" (for low),
6. Replaces 'nnnL2' with "nnn@M" (for medium),
7. Replaces 'nnnL3' with 'nnn@H" (for high)

buildcache.py

@@ -28,10 +28,111 @@
 from tradeexcept import TradeException
 
 # Find the non-comment part of a string
-noCommentRe = re.compile(r'^\s*(?P<text>(?:[^\\#]|\\.)+?)\s*(#|$)')
+noCommentRe = re.compile(r'^\s*(?P<text>(?:[^\\#]|\\.)*)\s*(#|$)')
 systemStationRe = re.compile(r'^\@\s*(.*)\s*/\s*(.*)')
 categoryRe = re.compile(r'^\+\s*(.*?)\s*$')
-itemPriceRe = re.compile(r'^(.*?)\s+(\d+)\s+(\d+)(?:\s+(\d{4}-.*?)(?:\s+demand\s+(-?\d+)L(-?\d+)\s+stock\s+(-?\d+)L(-?\d+))?)?$')
+
+# first part of any prices line is the item name and paying/asking price
+itemPriceFrag = r"""
+    # match item name, allowing spaces in the name
+    (?P<item> .*?)
+\s+
+    # price station is buying the item for
+    (?P<paying> \d+)
+\s+
+    # price station is selling item for
+    (?P<asking> \d+)
+"""
+
+# time formats per https://www.sqlite.org/lang_datefunc.html
+# YYYY-MM-DD HH:MM:SS
+# YYYY-MM-DDTHH:MM:SS
+# HH:MM:SS
+# 'now'
+timeFrag = r'(?P<time>(\d{4}-\d{2}-\d{2}[T ])?\d{2}:\d{2}:\d{2}|now)'
+
+# format used with --full and in TradeDangerous.prices
+# <item name> <paying> <asking> [ <time> [ demand <units>L<level> stock <units>L<level> ] ]
+itemPriceRe = re.compile(r"""
+^
+    # name, prices
+    {base_f}
+    # extended section with time and possibly demand+stock info
+    (?:
+    \s+
+        {time_f}
+        # optional demand/stock after time
+        (?:
+            \s+ demand \s+ (?P<demand> -?\d+L-?\d+ | n/a | -L-)
+            \s+ stock \s+ (?P<stock> -?\d+L-?\d+ | n/a | -L-)
+        )?
+    )?
+\s*
+$
+""".format(base_f=itemPriceFrag, time_f=timeFrag), re.IGNORECASE + re.VERBOSE)
+
+# new format: <name> <paying> <asking> [ <demUnits>@<demLevel> <stockUnits>@<stockLevel> [ <time> | now ] ]
+qtyLevelFrag = r"""
+    unk                         # You can just write 'unknown'
+|   n/a                         # alias for 0@0
+|   -                           # alias for 0@0
+|   \?@\?                       # Or ?@?
+|   \d+@[LMH]                   # Or <number>@<level> where level is L(ow), M(ed) or H(igh)
+"""
+newItemPriceRe = re.compile(r"""
+^
+    {base_f}
+\s+ 
+    # demand units and level
+    (?P<demand> {qtylvl_f})
+\s+
+    # stock units and level
+    (?P<stock> {qtylvl_f})
+    # time is optional
+    (?:
+    \s+
+        {time_f}
+    )?
+\s*
+$
+""".format(base_f=itemPriceFrag, qtylvl_f=qtyLevelFrag, time_f=timeFrag), re.IGNORECASE + re.VERBOSE)
+
+
+class UnitsAndLevel(object):
+    """
+        Helper class for breaking a units-and-level reading (e.g. -1L-1 or 50@M)
+        into units and level values or throwing diagnostic messages to help the
+        user figure out what data error was made.
+    """
+    # Map textual representations of levels back into integer values
+    levels = {
+        '-1': -1,
+        '0': 0, '-': 0,
+        'L': 1, '1': 1,
+        'M': 2, '2': 2,
+        'H': 3, '3': 3,
+    }
+    # Split a <units>L<level> reading
+    splitLRe = re.compile(r'^(?P<units>\d+)L(?P<level>\d+)$')
+    # Split a <units>@<level> reading
+    splitAtRe = re.compile(r'^(?P<units>\d+)@(?P<level>[LMH])$')
+
+    def __init__(self, category, reading):
+        if reading in (None, "unk", "?@?", "-1L-1"):
+            self.units, self.level = -1, -1
+        elif reading in ("-@-", "-L-", "-", "n/a"):
+            self.units, self.level = 0, 0
+        else:
+            matches = self.splitLRe.match(reading) or self.splitAtRe.match(reading)
+            if not matches:
+                raise ValueError("Invalid {} units/level value. Expected 'unk', <units>L<level> or <units>@[LMH], got '{}'".format(category, reading))
+            units, level = matches.group('units', 'level')
+            try:
+                self.units, self.level = int(units), UnitsAndLevel.levels[level]
+            except KeyError:
+                raise ValueError("Invalid {} level '{}' (expected 0 or - for unavailable, 1 or l for low, 2 or m for medium, 3 or h for high)".format(category, level))
+            if self.units < 0:
+                raise ValueError("Negative {} quantity '{}' specified, please use 'unk' for unknown".format(category, self.units))
 
 
 class UnknownItemError(TradeException):
@@ -113,20 +214,25 @@ def priceLineNegotiator(priceFile, db, debug=0):
 
             matches = itemPriceRe.match(text)
             if not matches:
-                print("Unrecognized line/syntax: {}".format(line))
-                sys.exit(1)
+                matches = newItemPriceRe.match(text)
+                if not matches:
+                    print("Unrecognized line/syntax: {}".format(line))
+                    sys.exit(1)
 
-            itemName, stationPaying, stationAsking, modified = matches.group(1), int(matches.group(2)), int(matches.group(3)), matches.group(4)
-            demand, demandLevel, stock, stockLevel = int(matches.group(5) or -1), int(matches.group(6) or -1), int(matches.group(7) or -1), int(matches.group(8) or -1)
+            itemName, stationPaying, stationAsking, modified = matches.group('item'), int(matches.group('paying')), int(matches.group('asking')), matches.group('time')
+            demand = UnitsAndLevel('demand', matches.group('demand'))
+            stock  = UnitsAndLevel('stock',  matches.group('stock'))
+            if modified and modified.lower() in ('now', '"now"', "'now'"):
+                modified = None         # Use CURRENT_FILESTAMP
 
             try:
                 itemID = itemsByName["{}:{}".format(categoryID, itemName)] if qualityItemWithCategory else itemsByName[itemName]
             except KeyError as key:
                 raise UnknownItemError(priceFile, lineNo, key)
 
             uiOrder += 1
-            yield PriceEntry(stationID, itemID, stationPaying, stationAsking, uiOrder, modified, demand, demandLevel, stock, stockLevel)
-        except (AttributeError, IndexError):
+            yield PriceEntry(stationID, itemID, stationPaying, stationAsking, uiOrder, modified, demand.units, demand.level, stock.units, stock.level)
+        except UnknownItemError:
             continue
 
 

data/prices.py → prices.py

@@ -15,7 +15,7 @@
 ######################################################################
 # Main
 
-def dumpPrices(dbFilename, withModified=False, stationID=None, file=None, defaultZero=False, debug=0):
+def dumpPrices(dbFilename, withModified=False, withLevels=False, stationID=None, file=None, defaultZero=False, debug=0):
     """ Generate a 'prices' list for the given list of stations using data from the DB. """
     conn = sqlite3.connect(str(dbFilename))     # so we can handle a Path object too
     cur  = conn.cursor()
@@ -33,21 +33,17 @@ def dumpPrices(dbFilename, withModified=False, stationID=None, file=None, defaul
         # check if there are prices for the station
         cur.execute("SELECT COUNT(*) FROM Price WHERE Price.station_id = {}".format(stationID))
         priceCount = cur.fetchone()[0]
-        # generate new timestamp in the select
-        modifiedStamp = "CURRENT_TIMESTAMP"
     else:
         # no station, no check
         priceCount = 1
-        # use old timestamp for the export
-        modifiedStamp = "Price.modified"
 
     stationClause = "1" if not stationID else "Station.station_id = {}".format(stationID)
     defaultDemandVal = 0 if defaultZero else -1
     if priceCount == 0:
         # no prices, generate an emtpy one with all items
         cur.execute("""
             SELECT  Station.system_id, Station.station_id, Item.category_id, Item.item_id,
-                    0, 0, CURRENT_TIMESTAMP,
+                    0, 0, NULL,
                     {defDemand}, {defDemand}, {defDemand}, {defDemand}
                FROM Item LEFT OUTER JOIN Station, Category
               WHERE {stationClause}
@@ -62,7 +58,7 @@ def dumpPrices(dbFilename, withModified=False, stationID=None, file=None, defaul
                     , Price.item_id
                     , Price.sell_to
                     , Price.buy_from
-                    , {modStamp}  -- real or current timestamp
+                    , Price.modified
                     , IFNULL(Price.demand, {defDemand})
                     , IFNULL(Price.demand_level, {defDemand})
                     , IFNULL(Price.stock, {defDemand})
@@ -72,7 +68,7 @@ def dumpPrices(dbFilename, withModified=False, stationID=None, file=None, defaul
                     AND Station.station_id = Price.station_id
                     AND (Item.category_id = Category.category_id) AND Item.item_id = Price.item_id
              ORDER  BY Station.system_id, Station.station_id, Category.name, Price.ui_order, Item.name
-        """.format(modStamp=modifiedStamp, stationClause=stationClause, defDemand=defaultDemandVal))
+        """.format(stationClause=stationClause, defDemand=defaultDemandVal))
 
     lastSys, lastStn, lastCat = None, None, None
 
@@ -84,18 +80,54 @@ def dumpPrices(dbFilename, withModified=False, stationID=None, file=None, defaul
         file.write("# TradeDangerous prices for ALL Systems/Stations\n")
     file.write("\n")
 
-    file.write("# The order items are listed in is saved to the DB,\n")
-    file.write("# feel free to move items around within their categories.\n")
+    file.write("# REMOVE ITEMS THAT DON'T APPEAR IN THE UI\n")
+    file.write("# ORDER IS REMEMBERED: Move items around within categories to match the game UI\n")
     file.write("\n")
 
-    if not withModified:
-        file.write("# <item name> <sell> <buy>\n")
-    else:
-        file.write("# <item name> <sell> <buy> <timestamp> demand <demand#>L<level> stock <stock#>L<level>\n")
-        file.write("#  demand#/stock#: the quantity available or -1 for 'unknown'")
-        file.write("#  level: 0 = None, 1 = Low, 2 = Medium, 3 = High, -1 = Unknown\n")
+    file.write("# File syntax:\n")
+    file.write("# <item name> <sell> <buy> [ <demand units>@<demand level> <stock units>@<stock level> [<timestamp>] ]\n")
+    file.write("# You can write 'unk' for unknown demand/stock, 'n/a' if the item is unavailable,\n")
+    file.write("# level can be one of 'L', 'M' or 'H'.\n")
+    file.write("# If you omit the timestamp, the current time will be used when the file is loaded.\n")
+
+    if defaultZero:
+        file.write("\n")
+        file.write("# CAUTION: Items marked 'n/a' are ignored for trade planning.\n")
+
     file.write("\n")
 
+    levelDescriptions = {
+        -1: "?",
+         0: "0",
+         1: "L",
+         2: "M",
+         3: "H"
+    }
+    def itemQtyAndLevel(quantity, level):
+        if defaultZero and quantity == -1 and level == -1:
+            quantity, level = 0, 0
+        if quantity < 0 and level < 0:
+            return "      unk"
+        if quantity == 0 and level == 0:
+            return "      n/a"
+        # Quantity of -1 indicates 'unknown'
+        quantityDesc = '?' if quantity < 0 else str(quantity)
+        # try to use a descriptive for the level
+        try:
+            levelDesc = levelDescriptions[int(level)]
+        except (KeyError, ValueError):
+            levelDesc = str(level)
+        return "{:>7}@{}".format(quantityDesc, levelDesc)
+
+
+    maxCrWidth = 7
+    file.write("#     {:<{width}} {:>{crwidth}} {:>{crwidth}}".format("Item Name", "Sell Cr", "Buy Cr", width=longestNameLen, crwidth=maxCrWidth))
+    if withLevels:
+        file.write("  {:>9} {:>9}".format("Demand", "Stock"))
+    if withModified:
+        file.write("  {}".format("Modified"))
+    file.write("\n\n")
+
     for (sysID, stnID, catID, itemID, fromStn, toStn, modified, demand, demandLevel, stock, stockLevel) in cur:
         system = systems[sysID]
         if system is not lastSys:
@@ -115,25 +147,17 @@ def dumpPrices(dbFilename, withModified=False, stationID=None, file=None, defaul
             file.write("   + {}\n".format(category))
             lastCat = category
 
-        file.write("      {:<{width}} {:7d} {:6d}".format(items[itemID], fromStn, toStn, width=longestNameLen))
-        if withModified and modified:
-            if defaultZero:
-                if demand == -1 and demandLevel == -1:
-                    demand      = 0
-                    demandLevel = 0
-                if stock == -1 and stockLevel == -1:
-                    stock      = 0
-                    stockLevel = 0
-            file.write("   {} demand {:>7}L{} stock {:>7}L{}".format(
-                        modified,
-                        demand,
-                        demandLevel,
-                        stock,
-                        stockLevel
+        file.write("      {:<{width}} {:{crwidth}d} {:{crwidth}d}".format(items[itemID], fromStn, toStn, width=longestNameLen, crwidth=maxCrWidth))
+        if withLevels:
+            file.write("  {} {}".format(
+                        itemQtyAndLevel(demand, demandLevel),
+                        itemQtyAndLevel(stock, stockLevel),
                     ))
+        if withModified:
+            file.write("  {}".format(modified or 'now'))
         file.write("\n")
 
 
 if __name__ == "__main__":
     from tradedb import TradeDB
-    dumpPrices(TradeDB.defaultDB, withModified=True)
+    dumpPrices(TradeDB.defaultDB, withModified=True, withLevels=True)

trade.py

@@ -607,7 +607,9 @@ def editUpdate(args, stationID):
         with tmpPath.open("w") as tmpFile:
             # Remember the filename so we know we need to delete it.
             absoluteFilename = str(tmpPath.resolve())
-            prices.dumpPrices(args.db, withModified=args.all, file=tmpFile, stationID=stationID, defaultZero=args.zero, debug=args.debug)
+            withModified = args.all # or args.timestamps
+            withLevels   = args.all # or args.levels
+            prices.dumpPrices(args.db, withModified=withModified, withLevels=withLevels, file=tmpFile, stationID=stationID, defaultZero=args.zero, debug=args.debug)
 
         # Stat the file so we can determine if the user writes to it.
         # Use the most recent create/modified timestamp.
@@ -648,7 +650,7 @@ def editUpdate(args, stationID):
             print("# Update complete, regenerating .prices file")
 
         with tdb.pricesPath.open("w") as pricesFile:
-            prices.dumpPrices(args.db, withModified=True, file=pricesFile, debug=args.debug)
+            prices.dumpPrices(args.db, withModified=True, withLevels=True, file=pricesFile, debug=args.debug)
 
         # Update the DB file so we don't regenerate it.
         pathlib.Path(args.db).touch()