support for demand/stock levels in emdn data and weeding out fake ent…

…ries caused by the way UI shows mission items

data/TradeDangerous.sql

@@ -362,10 +362,10 @@ CREATE TABLE Price
    -- how many credits must you pay to buy at this station?
    buy_from INTEGER NOT NULL DEFAULT 0,
    modified DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
-   demand INTEGER,
-   demandLevel INTEGER,
-   stock INTEGER,
-   stockLevel INTEGER,
+   demand INTEGER DEFAULT -1,
+   demand_level INTEGER DEFAULT -1,
+   stock INTEGER DEFAULT -1,
+   stock_level INTEGER DEFAULT -1,
 
    PRIMARY KEY (item_id, station_id),
 

data/buildcache.py

@@ -28,9 +28,9 @@
 noCommentRe = re.compile(r'^\s*(?P<text>(?:[^\\#]|\\.)+?)\s*(#|$)')
 systemStationRe = re.compile(r'^\@\s*(.*)\s*/\s*(.*)')
 categoryRe = re.compile(r'^\+\s*(.*?)\s*$')
-itemPriceRe = re.compile(r'^(.*?)\s+(\d+)\s+(\d+)(?:\s+(\d{4}-.*?))?$')
+itemPriceRe = re.compile(r'^(.*?)\s+(\d+)\s+(\d+)(?:\s+(\d{4}-.*?)(?:\s+demand\s+(-?\d+)L(-?\d+)\s+stock\s+(-?\d+)L(-?\d+))?)?$')
 
-class PriceEntry(namedtuple('PriceEntry', [ 'stationID', 'itemID', 'asking', 'paying', 'uiOrder', 'modified' ])):
+class PriceEntry(namedtuple('PriceEntry', [ 'stationID', 'itemID', 'asking', 'paying', 'uiOrder', 'modified', 'demand', 'demandLevel', 'stock', 'stockLevel' ])):
     pass
 
 def priceLineNegotiator(priceFile, db, debug=0):
@@ -89,9 +89,10 @@ def priceLineNegotiator(priceFile, db, debug=0):
                 print("Unrecognized line/syntax: {}".format(line))
                 sys.exit(1)
             itemName, stationPaying, stationAsking, modified = matches.group(1), int(matches.group(2)), int(matches.group(3)), matches.group(4)
+            demand, demandLevel, stock, stockLevel = int(matches.group(5) or -1), int(matches.group(6) or -1), int(matches.group(7) or -1), int(matches.group(8) or -1)
             itemID = itemsByName["{}:{}".format(categoryID, itemName)]
             uiOrder += 1
-            yield PriceEntry(stationID, itemID, stationPaying, stationAsking, uiOrder, modified)
+            yield PriceEntry(stationID, itemID, stationPaying, stationAsking, uiOrder, modified, demand, demandLevel, stock, stockLevel)
         except (AttributeError, IndexError):
             continue
 
@@ -111,8 +112,8 @@ def processPricesFile(db, pricesPath, stationID=None, debug=0):
             if debug > 2: print(price)
             bindValues += [ price ]
         stmt = """
-                   INSERT INTO Price (station_id, item_id, sell_to, buy_from, ui_order, modified)
-                   VALUES (?, ?, ?, ?, ?, IFNULL(?, CURRENT_TIMESTAMP))
+                   INSERT OR REPLACE INTO Price (station_id, item_id, sell_to, buy_from, ui_order, modified, demand, demand_level, stock, stock_level)
+                   VALUES (?, ?, ?, ?, ?, IFNULL(?, CURRENT_TIMESTAMP), ?, ?, ?, ?)
                 """
         db.executemany(stmt, bindValues)
     db.commit()

data/prices.py

@@ -38,6 +38,10 @@ def dumpPrices(dbFilename, withModified=False, stationID=None, file=None, debug=
                 , Price.sell_to
                 , Price.buy_from
                 , Price.modified
+                , IFNULL(Price.demand, -1)
+                , IFNULL(Price.demand_level, -1)
+                , IFNULL(Price.stock, -1)
+                , IFNULL(Price.stock_level, -1)
           FROM  Station, Item, Category, Price
          WHERE  {}  -- station clause
                 AND Station.station_id = Price.station_id
@@ -49,7 +53,7 @@ def dumpPrices(dbFilename, withModified=False, stationID=None, file=None, debug=
     if not file: file = sys.stdout
     file.write("# Source for TradeDangerous' price database.\n\n")
 
-    for (sysID, stnID, catID, itemID, fromStn, toStn, modified) in cur:
+    for (sysID, stnID, catID, itemID, fromStn, toStn, modified, demand, demandLevel, stock, stockLevel) in cur:
         system = systems[sysID]
         if system is not lastSys:
             if lastStn: file.write("\n\n")
@@ -70,7 +74,13 @@ def dumpPrices(dbFilename, withModified=False, stationID=None, file=None, debug=
 
         file.write("      {:<{width}} {:7d} {:6d}".format(items[itemID], fromStn, toStn, width=longestNameLen))
         if withModified and modified:
-            file.write("   {}".format(modified))
+            file.write("   {} demand {:>7}L{} stock {:>7}L{}".format(
+                        modified,
+                        demand,
+                        demandLevel,
+                        stock,
+                        stockLevel
+                    ))
         file.write("\n")
 
 

emdn-tap.py

@@ -157,8 +157,8 @@ def bleat(category, name, *args, **kwargs):
 # Save data to the db
 
 commitStmt = """
-    INSERT OR REPLACE INTO Price (item_id, station_id, ui_order, sell_to, buy_from, modified)
-    VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
+    INSERT OR REPLACE INTO Price (item_id, station_id, ui_order, sell_to, buy_from, modified, demand, demand_level, stock, stock_level)
+    VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ?, ?)
 """
 
 def commit(tdb, db, recordsSinceLastCommit, pargs):
@@ -172,7 +172,7 @@ def commit(tdb, db, recordsSinceLastCommit, pargs):
 
     if pargs.verbose:
         print("-> commit {} records".format(len(recordsSinceLastCommit)) + (" [disabled]" if pargs.noWrites else ""))
-        if pargs.verbose > 2:
+        if pargs.verbose > 3:
             print("\n".join(['#  {}'.format(str(items)) for items in recordsSinceLastCommit]))
     if not pargs.noWrites:
         # Save the records.
@@ -221,14 +221,35 @@ def main():
                     recordsSinceLastCommit = []
                     lastCommit = now
 
+            if rec.payingCr == 0 and rec.askingCr == 0:
+                if pargs.verbose > 2:
+                    print("# Ignoring 0/0 entry for {} @ {}/{}".format(rec.item, rec.system, rec.station))
+            if rec.payingCr < 0 or rec.askingCr < 0 \
+            or rec.stock < 0 or rec.stockLevel < 0 \
+            or rec.demand < 0 or rec.demandLevel < 0:
+                bleat("item", '{}@{}/{}'.format(rec.item, rec.system, rec.station), "Invalid (negative) value in price/stock fields")
+                continue
+
             records += 1
 
             if pargs.verbose and (records % 1000 == 0):
                 print("# At {} captured {} records.".format(rec.timestamp, records))
+
             if pargs.verbose > 1:
                 paying = localedNo(rec.payingCr)+'cr' if rec.payingCr else '    -    '
                 asking = localedNo(rec.askingCr)+'cr' if rec.askingCr else '    -    '
-                print("{} {:.<65} {:>9} {:>9}".format(rec.timestamp, '{} @ {}/{}'.format(rec.item, rec.system, rec.station), paying, asking))
+                desc = '{} @ {}/{}'.format(rec.item, rec.system, rec.station)
+                extra = " | {:>6}L{} {:>6}L{}".format(rec.demand, rec.demandLevel, rec.stock, rec.stockLevel) if pargs.verbose > 2 else ""
+                print("{} {:.<65} {:>9} {:>9}{}".format(rec.timestamp, desc, paying, asking, extra))
+
+            # As of Beta 1.04, if you are carrying an item that the station doesn't handle
+            # the UI shows a fake entry with the prices from the station you bought the
+            # item from.
+
+            if rec.demandLevel == 0 and rec.stockLevel == 0:
+                if pargs.verbose > 2:
+                    print("# Ignoring fake entry for {} @ {}/{}".format(rec.item, rec.system, rec.station))
+                continue
 
             # Find the item in the price database to get its data and make sure
             # it matches the category we expect to see it listed in.
@@ -254,7 +275,7 @@ def main():
                 continue
 
             uiOrder = getItemUIOrder(station.ID, item.category.ID, item.ID)
-            recordsSinceLastCommit.append([ item.ID, station.ID, uiOrder, rec.payingCr, rec.askingCr ])
+            recordsSinceLastCommit.append([ item.ID, station.ID, uiOrder, rec.payingCr, rec.askingCr, rec.demand, rec.demandLevel, rec.stock, rec.stockLevel ])
     except KeyboardInterrupt:
         print("Ctrl-C pressed, stopping.")
 

emdn/firehose.py

@@ -166,6 +166,7 @@ def drink(self, records=None, timeout=None, burst=False):
                 while recordsRemaining:
                     try:
                         jsData = self.read(nonBlocking=True)
+                        if self.debug > 3: print(jsData)
                     except EOFError:
                         return
                     except (zmq.error.Again, BlockingIOError, InterruptedError):