Merged kfsone/tradedangerous into master

.gitignore

@@ -11,6 +11,7 @@ data/TradeDangerous.prices
 data/*.stamp
 data/*.ini
 data/extra-stars.txt
+data/*.txt
 *.prices
 *.suo
 *.pyperf

CHANGES.txt

@@ -2,6 +2,55 @@
 TradeDangerous, Copyright (C) Oliver "kfsone" Smith, July 2014
 ==============================================================================
 
+v6.16.1 Apr 01 2015 (In some parts of the world)
+. (kfsone) "old" sub-command
+   - Added a "--route" option: which probably doesn't do what you think,
+   - Added a "--min-age" option: because old is the new new!
+
+v6.16.0 Mar 29 2015
+. (tKe) "run" command
+   - "--loop" option
+     Looks for routes which loop back to the origin station,
+   - Optimizations,
+   - Minor bug fix in destination culling,
+
+v6.15.0 Mar 27 2015
+. (kfsone) Refresh .prices file after removing a station that had items
+. (kfsone) Maddavo Plugin
+   - Added support for "corrections"
+     "--opt=corrections" and "--opt=csvs" will read Maddavo's correction data
+     correction data is read first and is applied to existing data,
+   - Support for Category, Item, System, Station, Ship, ShipVendor and
+     RareItem name changes and deletions,
+   - System, Item and Station deletions can be very slow if there are prices
+     in the database, be patient,
+   - "--opt=csvs" now also enables "exportcsvs"
+   - "--opt=exportcsvs" not exports 
++ Over 100 additional Systems
+API:
+. (kfsone) transfers.CSVStream will give a line-by-line report of utf-8
+     decode errors but continue streaming.
+
+v6.14.6 Mar 25 2015
+. (kfsone) Item names in .price files are now case insensitive (Ai Relics)
++ Over 100 systems
+
+v6.14.5 Mar 23 2015
+. (kfsone) Solved the missing 6.14.3 commits
+
+v6.14.4 Mar 22 2015
+. (gazelle) Bernd's "getsation.sh" script (see scripts/README.txt)
+. (kfsone) Fixed #209 Exception when you couldn't afford something,
+. (kfsone) Fixed #210 Exception when --from not reached,
+. (kfsone) Added "Salvage" Category,
+. (kfsone) Added "AI Relics" and "Antiquities" Items,
+ - You will want to "trade.py buildcache -f -i"
+
+v6.14.3 Mar 17 2015
+. (kfsone) Windows improvements to misc/madupload.py,
+. (kfsone) EDDB v3 support,
+. (kfsone) "--towards" should be much better behaved
+
 v6.14.2 Mar 15 2015
 . (kfsone) Minor tweaks
 . (kfsone) Removed the "tradingWith" cache because its hit rate was so low

cache.py

@@ -473,7 +473,7 @@ def getItemByNameIndex(cur):
     """
         Generate item name index.
     """
-    cur.execute("SELECT item_id, name FROM item")
+    cur.execute("SELECT item_id, UPPER(name) FROM item")
     return { name: itemID for (itemID, name) in cur }
 
 
@@ -650,29 +650,28 @@ def changeStation(matches):
         )
 
     addItem, addBuy, addSell = items.append, buys.append, sells.append
+    getItemID = itemByName.get
 
     def processItemLine(matches):
         itemName, modified = matches.group('item', 'time')
+        itemName = itemName.upper()
 
         # Look up the item ID.
-        try:
-            itemID = itemByName[itemName]
-        except KeyError:
-            itemID = -1
+        itemID = getItemID(itemName, -1)
         if itemID < 0:
             oldName = itemName
             itemName = corrections.correctItem(itemName)
             if itemName == DELETED:
                 DEBUG1("DELETED {}", oldName)
                 return
-            try:
-                itemID = itemByName[itemName]
-                DEBUG1("Renamed {} -> {}", oldName, itemName)
-            except KeyError:
+            itemName = itemName.upper()
+            itemID = getItemID(itemName, -1)
+            if itemID < 0:
                 ignoreOrWarn(
                     UnknownItemError(priceFile, lineNo, itemName)
                 )
                 return
+            DEBUG1("Renamed {} -> {}", oldName, itemName)
 
         # Check for duplicate items within the station.
         if itemID in processedItems:
@@ -822,7 +821,7 @@ def processPricesFile(tdenv, db, pricesPath, pricesFh=None, defaultZero=False):
 
     tdenv.NOTE(
             "Import complete: "
-                "{:n} items ({:n} buyers, {:n} sellers) "
+                "{:n} items ({:n} buy, {:n} sell) "
                 "for {:n} stations "
                 "in {:n} systems",
                     len(items),

commands/olddata_cmd.py

@@ -5,6 +5,7 @@
 
 import itertools
 import math
+import sys
 
 ######################################################################
 # Parser config
@@ -32,6 +33,15 @@
             metavar='N.NN',
             type=float,
     ),
+    ParseArgument('--route',
+            help='Sort to shortest path',
+            action='store_true',
+    ),
+    ParseArgument('--min-age',
+            help='List data older than this number of days.',
+            type=float,
+            dest='minAge',
+    ),
 ]
 
 ######################################################################
@@ -62,6 +72,12 @@ def run(results, cmdenv, tdb):
     wheres = []
     havings = []
 
+    if cmdenv.minAge:
+        wheres.append(
+            "(JULIANDAY('NOW') - JULIANDAY(si.modified) >= {})"
+            .format(cmdenv.minAge)
+        )
+
     nearSys = cmdenv.nearSystem
     if nearSys:
         maxLy = cmdenv.maxLyPer or tdb.maxSystemLinkLy
@@ -141,6 +157,34 @@ def run(results, cmdenv, tdb):
         row.dist = dist2 ** 0.5
         results.rows.append(row)
 
+    if cmdenv.near:
+        results.rows.sort(key=lambda row: row.dist)
+
+    if cmdenv.route and len(results.rows) > 1:
+        def walk(startNode, dist):
+            rows = results.rows
+            startNode = rows[startNode]
+            openList = set(rows)
+            path = [startNode]
+            openList.remove(startNode)
+            while len(path) < len(rows):
+                lastNode = path[-1]
+                distFn = lastNode.station.system.distanceTo
+                nearest = min(openList, key=lambda row: distFn(row.station.system))
+                openList.remove(nearest)
+                path.append(nearest)
+                dist += distFn(nearest.station.system)
+            return (path, dist)
+        if cmdenv.near:
+            bestPath = walk(0, results.rows[0].dist)
+        else:
+            bestPath = (results.rows, float("inf"))
+            for i in range(len(results.rows)):
+                path = walk(i, 0)
+                if path[1] < bestPath[1]:
+                    bestPath = path
+        results.rows[:] = bestPath[0]
+
     return results
 
 ######################################################################

commands/run_cmd.py

@@ -51,6 +51,11 @@
                 metavar='SYSTEM',
                 default=None,
         ),
+        ParseArgument('--loop',
+                help='Return to the starting station.',
+                action='store_true',
+                default=False,
+        ),
     ),
     ParseArgument('--via',
             help='Require specified systems/stations to be en-route.',
@@ -120,7 +125,7 @@
             dest='maxAge',
         ),
     ParseArgument('--pad-size', '-p',
-            help='Limit the padsize to this ship size (S,M,L or ? for unkown).',
+            help='Limit the padsize to this ship size (S,M,L or ? for unknown).',
             metavar='PADSIZES',
             dest='padSize',
         ),
@@ -734,6 +739,12 @@ def validateRunArguments(tdb, cmdenv, calc):
         if cmdenv.insurance >= (cmdenv.credits + arbitraryInsuranceBuffer):
             raise CommandLineError("Insurance leaves no margin for trade")
 
+    if cmdenv.loop:
+        if cmdenv.unique:
+            raise CommandLineError("Cannot use --unique and --loop together")
+        if cmdenv.direct:
+            raise CommandLineError("Cannot use --direct and --loop together")
+
     checkOrigins(tdb, cmdenv, calc)
     checkDestinations(tdb, cmdenv, calc)
 
@@ -788,7 +799,7 @@ def validateRunArguments(tdb, cmdenv, calc):
         ]
         cmdenv.destinations = [
             dest for dest in cmdenv.destinations
-            if destination.system in viaSystems
+            if dest.system in viaSystems
         ]
         cmdenv.destSystems = [
             dest.system for dest in cmdenv.destinations
@@ -904,8 +915,8 @@ def checkReachability(tdb, cmdenv):
 
             # Were there just not enough hops?
             jumpLimit = cmdenv.maxJumpsPer * cmdenv.hops
-            if jumpLimit < len(route):
-                routeJumps = len(route) - 1
+            routeJumps = len(route) - 1
+            if jumpLimit < routeJumps:
                 hopsRequired = math.ceil(routeJumps / cmdenv.maxJumpsPer)
                 jumpsRequired = math.ceil(routeJumps / cmdenv.hops)
                 raise CommandLineError(
@@ -985,6 +996,7 @@ def run(results, cmdenv, tdb):
     stopStations = cmdenv.destinations
     goalSystem = cmdenv.goalSystem
     maxLs = cmdenv.maxLs
+    maxHopDistLy = cmdenv.maxJumpsPer * cmdenv.maxLyPer
 
     # seed the route table with starting places
     startCr = cmdenv.credits - cmdenv.insurance
@@ -1011,13 +1023,19 @@ def run(results, cmdenv, tdb):
     results.summary.exception = ""
 
     pruneMod = cmdenv.pruneScores / 100
+    distancePruning = (cmdenv.destPlace and not cmdenv.direct) or (cmdenv.loop)
+    if distancePruning and not cmdenv.loop:
+        stopSystems = {stop.system for stop in stopStations}
 
+    loopRoutes = []
     for hopNo in range(numHops):
         restrictTo = None
         if hopNo == lastHop and stopStations:
             restrictTo = set(stopStations)
+            manualRestriction = bool(cmdenv.destPlace)
         elif len(viaSet) > cmdenv.adhocHops:
             restrictTo = viaSet
+            manualRestriction = True
 
         if hopNo >= 1 and cmdenv.maxRoutes or pruneMod:
             routes.sort()
@@ -1029,6 +1047,21 @@ def run(results, cmdenv, tdb):
             if cmdenv.maxRoutes and len(routes) > cmdenv.maxRoutes:
                 routes = routes[:cmdenv.maxRoutes]
 
+        if distancePruning:
+            remainingDistance = (numHops - hopNo) * maxHopDistLy
+            def routeStillHasAChance(r):
+                distanceFn = r.lastSystem.distanceTo
+                if cmdenv.loop:
+                    return distanceFn(r.firstSystem) <= remainingDistance
+                else:
+                    return any(stop for stop in stopSystems if distanceFn(stop) <= remainingDistance)
+
+            preCrop = len(routes)
+            routes[:] = [x for x in routes if routeStillHasAChance(x)]
+            pruned = preCrop - len(routes)
+            if pruned:
+                cmdenv.NOTE("Pruned {} origins too far from any end stations", pruned)
+
         if cmdenv.progress:
             print("* Hop {:3n}: {:.>10n} origins".format(hopNo+1, len(routes)))
         elif cmdenv.debug:
@@ -1053,7 +1086,7 @@ def run(results, cmdenv, tdb):
         if not newRoutes:
             checkReachability(tdb, cmdenv)
             if hopNo > 0:
-                if restrictTo:
+                if restrictTo and manualRestriction:
                     results.summary.exception += routeFailedRestrictions(
                         tdb, cmdenv, restrictTo, maxLs, hopNo
                     )
@@ -1098,8 +1131,21 @@ def run(results, cmdenv, tdb):
             )
             if routes[0].lastSystem is goalSystem:
                 cmdenv.NOTE("Goal system reached!")
+                routes = routes[:1]
                 break
 
+        if cmdenv.loop:
+            for route in routes:
+                if route.lastStation == route.firstStation:
+                    loopRoutes.append(route)
+
+    if cmdenv.loop:
+        routes = loopRoutes
+
+        # normalise the scores for fairness...
+        for route in routes:
+            route.score /= len(route.hops)
+
     if not routes:
         raise NoDataError(
             "No profitable trades matched your critera, "

commands/station_cmd.py

@@ -6,6 +6,7 @@
 from tradedb import System, Station
 from tradedb import TradeDB
 
+import cache
 import csvexport
 import difflib
 import re
@@ -287,6 +288,7 @@ def removeStation(tdb, cmdenv, station):
     db.commit()
     cmdenv.NOTE("{} (#{}) removed from {} database.",
             station.name(), station.ID, tdb.dbPath)
+    cmdenv.stationItemCount = station.itemCount
     return True
 
 
@@ -300,6 +302,12 @@ def checkResultAndExportStations(tdb, cmdenv, result):
 
     lines, csvPath = csvexport.exportTableToFile(tdb, cmdenv, "Station")
     cmdenv.NOTE("{} updated.", csvPath)
+
+    if cmdenv.remove:
+        if cmdenv.stationItemCount:
+            cmdenv.NOTE("Station had items, regenerating .prices file")
+            cache.regeneratePricesFile(tdb, cmdenv)
+
     return None
 
 

corrections.py

@@ -15,6 +15,8 @@
     "BODB DJEDI": "BODEDI",
     "HIP 101110": "NEW YEMBO",
     "ICV HR-V B2-0": "ICZ HR-V B2-0",
+    "FLECHS": "ICZ IR-W C1-13",
+    "PANDAMONIUM": "PANDEMONIUM",
 
 #ADD_SYSTEMS_HERE
 }
@@ -97,6 +99,14 @@
     "SAMBURITJA/RPTEYN LANDING":            DELETED,
     "CHICK EK/WH IEELOCK GATEWAY":          DELETED,
     "LHS 3006/WCM TRANSFER ORBITAL":        "Leonard Nimoy Station",
+    "LALANDE 4141/4A5O4O":                  "4A5O4D",
+    "NEW YEMBO/UNDER CONSTRUCTION":         DELETED,
+    "GENDINI/Houtman Gateway":              DELETED,
+    "QUECHE/Rutherford Works":              DELETED,
+    "SALARHUL/Lavoisier Outpost":           DELETED,
+    "TELIU YUAN/Buchli Station":            DELETED,
+    "WUNJO/Baudin Hanger":                  "Baudin Hangar",
+    "WYRD/Vonarburg Co-Operative":          "Vonarburg Co-operative",
 }
 
 categories = {

data/Category.csv

@@ -8,6 +8,7 @@ unq:name
 'Medicines'
 'Metals'
 'Minerals'
+'Salvage'
 'Slavery'
 'Technology'
 'Textiles'

data/Item.csv

@@ -58,6 +58,8 @@ unq:[email protected]_id,unq:name,ui_order
 'Minerals','Painite',7
 'Minerals','Rutile',8
 'Minerals','Uraninite',9
+'Salvage','AI Relics',1
+'Salvage','Antiquities',2
 'Slavery','Imperial Slaves',1
 'Slavery','Slaves',2
 'Technology','Advanced Catalysers',1