Merged kfsone/tradedangerous into master

.gitignore

@@ -8,6 +8,7 @@ tmp
 data/TradeDangerous.db
 data/TradeDangerous.db-journal
 data/TradeDangerous.prices
+data/*.stamp
 *.prices
 *.suo
 *.pyperf

CHANGES.txt

@@ -2,7 +2,26 @@
 TradeDangerous, Copyright (C) Oliver "kfsone" Smith, July 2014
 ==============================================================================
 
-v6.2.2 [wip]
+v6.2.3 Dec 17 2014
+. (kfsone) "maddavo" import plugin:
+            . --opt=skipdl will use previous downloads
+            . added timestamp tracking/checking (data/maddavo.stamp)
+            . don't rebuild the cache if nothing new was downloaded,
+            . use "-v" to see stats on what updates you've downloaded,
+            . use "-q" to see less output when using the plugin
+. (kfsone) Changed "nav" to show station count with "-v" instead of "--stations"
+. (kfsone) Removed the StationLink table - cache builds should be MUCH faster
+
+v6.2.2 Dec 17 2014
+. (kfsone) Removed Alloys, Combat Stabilisers, Cotton and Plastics
+  [there was no entry for them anywhere in the price database]
+. (kfsone) Plugins can now have their own options
+    maddavo's plugin will take --option=syscsv and
+      --option=stncsv to download those .csv files
+. (kfsone) Added "--download" option to "import" to stop after downloading
+. (kfsone) Added "--url" option to "import", e.g.
+    trade.py import --download --url http://kfs.org/td/prices kfs.prices
+. (kfsone) "--check --mfd" should now work if you have 64-bit drivers
 . (kfsone) Added "--max-days" to run command
 + Stations, ships: kfsone
 

cache.py

@@ -719,6 +719,8 @@ def processImportFile(tdenv, db, importPath, tableName):
         uniqueIndex = dict()
 
         for linein in csvin:
+            if not linein:
+                continue
             lineNo = csvin.line_num
             if len(linein) == columnCount:
                 tdenv.DEBUG1("       Values: {}", ', '.join(linein))
@@ -773,31 +775,6 @@ def processImportFile(tdenv, db, importPath, tableName):
                             table=tableName)
 
 
-def generateStationLink(tdenv, db):
-    tdenv.DEBUG0("Generating StationLink table")
-    db.create_function("sqrt", 1, math.sqrt)
-    db.execute("""
-            INSERT INTO StationLink
-            SELECT  lhs.system_id AS lhs_system_id,
-                    lhs.station_id AS lhs_station_id,
-                    rhs.system_id AS rhs_system_id,
-                    rhs.station_id AS rhs_station_id,
-                    sqrt(
-                        ((lSys.pos_x - rSys.pos_x) * (lSys.pos_x - rSys.pos_x)) +
-                        ((lSys.pos_y - rSys.pos_y) * (lSys.pos_y - rSys.pos_y)) +
-                        ((lSys.pos_z - rSys.pos_z) * (lSys.pos_z - rSys.pos_z))
-                        ) AS dist
-              FROM  Station AS lhs
-                    INNER JOIN System AS lSys
-                        ON (lhs.system_id = lSys.system_id),
-                    Station AS rhs
-                    INNER JOIN System AS rSys
-                        ON (rhs.system_id = rSys.system_id)
-              WHERE
-                    lhs.station_id != rhs.station_id
-    """)
-    db.commit()
-
 
 ######################################################################
 
@@ -854,10 +831,9 @@ def buildCache(tdb, tdenv):
     elif not tdenv.quiet:
         print("NOTE: Missing \"{}\" file - no price data".format(
                     str(pricesPath)
-                ), file=sys.stderr)
-
-    generateStationLink(tdenv, tempDB)
+            ), file=sys.stderr)
 
+    tempDB.commit()
     tempDB.close()
 
     tdenv.DEBUG0("Swapping out db files")

commands/export_cmd.py

@@ -35,7 +35,7 @@
 ]
 
 # some tables are ignored
-ignoreList = [ 'StationLink',
+ignoreList = [
 ]
 
 ######################################################################

commands/import_cmd.py

@@ -42,6 +42,16 @@
                 default=None,
         ),
     ),
+    ParseArgument('--url',
+        help='Name of the file to read.',
+        type=str,
+        default=None,
+    ),
+    ParseArgument('--download',
+        help='Stop after downloading.',
+        action='store_true',
+        default=False,
+    ),
     ParseArgument(
         '--ignore-unknown', '-i',
         default=False, action='store_true',
@@ -51,6 +61,14 @@
             "recognized is reported as warning but skipped."
         ),
     ),
+    ParseArgument(
+        '--option', '-O',
+        default=[], action='append',
+        dest='pluginOptions',
+        help=(
+            "Provides a way to pass additional arguments to plugins."
+        ),
+    ),
 ]
 
 ######################################################################
@@ -89,6 +107,8 @@ def run(results, cmdenv, tdb):
     if cmdenv.url:
         cmdenv.filename = cmdenv.filename or "import.prices"
         transfers.download(cmdenv, cmdenv.url, cmdenv.filename)
+        if cmdenv.download:
+            return None
 
     # If the filename specified was "-" or None, then go ahead
     # and present the user with an open file dialog.

commands/nav_cmd.py

@@ -27,12 +27,6 @@
                  'the system that station is in will be avoided instead.',
             action='append',
         ),
-    ParseArgument('--stations',
-            help='Show system\'s stations',
-            dest='showstations',
-        action='store_true',
-            default=False
-        ),
 ]
 
 ######################################################################
@@ -188,7 +182,7 @@ def render(results, cmdenv, tdb):
             key=lambda row: row.system.name())
     rowFmt.addColumn("JumpLy", '>', '7', '.2f',
             key=lambda row: row.jumpLy)
-    if cmdenv.showstations:
+    if cmdenv.detail:
         rowFmt.addColumn("Stations", '>', 2, 
             key=lambda row: len(row.system.stations))
     if cmdenv.detail:

commands/run_cmd.py

@@ -333,6 +333,7 @@ def validateRunArguments(tdb, cmdenv):
                 raise CommandLineError("Same to/from; more than one hop required.")
 
     viaSet = cmdenv.viaSet = set(cmdenv.viaPlaces)
+    cmdenv.DEBUG0("Via: {}", viaSet)
     viaSystems = set()
     for place in viaSet:
         if isinstance(place, Station):
@@ -345,6 +346,18 @@ def validateRunArguments(tdb, cmdenv):
         else:
             viaSystems.add(place)
 
+    avoids = cmdenv.avoidPlaces or []
+    for via in viaSet:
+        if isinstance(via, Station):
+            conflict = (via in avoids or via.system in avoids)
+        else:
+            conflict = (via in avoids)
+        if conflict:
+            raise CommandLineError(
+                    "Via {} conflicts with avoid list".format(
+                        via
+            ))
+
     # How many of the hops do not have pre-determined stations. For example,
     # when the user uses "--from", they pre-determine the starting station.
     fixedRoutePoints = 0
@@ -413,26 +426,55 @@ def validateRunArguments(tdb, cmdenv):
     if startStn:
         tdb.loadStationTrades([startStn.ID])
         if stopStn and cmdenv.hops == 1 and not stopStn in startStn.tradingWith:
-            raise CommandLineError("No profitable items found between {} and {}".format(
-                                startStn.name(), stopStn.name()))
+            raise CommandLineError(
+                    "No profitable items found between {} and {}".format(
+                        startStn.name(), stopStn.name()))
         if len(startStn.tradingWith) == 0:
-            raise NoDataError("No data found for potential buyers for items from {}.".format(
-                                startStn.name()))
+            raise NoDataError(
+                    "No data found for potential buyers for items from {}.".format(
+                        startStn.name()))
 
 
 ######################################################################
 
 
 def filterByVia(routes, viaSet, viaStartPos):
-    matchedRoutes = list(routes)
+    if not routes:
+        return []
+
+    matchedRoutes = []
+    partialRoutes = {}
+    maxMet = 0
     for route in routes:
         met = 0
         for hop in route.route[viaStartPos:]:
             if hop in viaSet or hop.system in viaSet:
                 met += 1
-        if met >= len(viaSet):
-            matchedRoutes.append(route)
-    return matchedRoutes
+        if met > 0:
+            if met >= len(viaSet):
+                matchedRoutes.append(route)
+            else:
+                if met > maxMet:
+                    partialRoutes[met] = []
+                if met >= maxMet:
+                    maxMet = met
+                    partialRoutes[met].append(route)
+
+    if matchedRoutes:
+        return matchedRoutes, None
+
+    if not maxMet:
+        raise NoDataError(
+                "No routes were found which matched your 'via' selections."
+        )
+
+    return partialRoutes[maxMet], (
+            "No runs visited all of your via destinations. "
+            "Listing runs that matched at least {}.".format(
+                    maxMet
+            )
+    )
+
 
 ######################################################################
 # Perform query and populate result set
@@ -501,36 +543,38 @@ def run(results, cmdenv, tdb):
         restrictTo = None
         if hopNo == lastHop and stopStations:
             restrictTo = stopStations
-            ### TODO:
-            ### if hopsLeft < len(viaSet):
-            ###   ... only keep routes that include len(viaSet)-hopsLeft routes
-            ### Thus: If you specify 5 hops via a,b,c and we are on hop 3, only keep
-            ### routes that include a, b or c. On hop 4, only include routes that
-            ### already include 2 of the vias, on hop 5, require all 3.
-            if viaSet:
-                routes = filterByVia(routes, viaSet, viaStartPos)
         elif len(viaSet) > cmdenv.adhocHops:
             restrictTo = viaSet
 
         routes = calc.getBestHops(routes, restrictTo=restrictTo)
 
-    if viaSet:
-        routes = filterByVia(routes, viaSet, viaStartPos)
+    results.summary = ResultRow()
+    results.summary.exception = None
 
     if not routes:
         raise NoDataError("No profitable trades matched your critera, or price data along the route is missing.")
 
+    if viaSet:
+        routes, caution = filterByVia(routes, viaSet, viaStartPos)
+        if caution:
+            results.summary.exception = caution
+
     routes.sort()
     results.data = routes
 
     return results
 
+
 ######################################################################
 # Transform result set into output
 
 def render(results, cmdenv, tdb):
     from formatting import RowFormat, ColumnFormat
 
+    exception = results.summary.exception
+    if exception:
+        print("\aCAUTION: {}\n".format(exception))
+
     routes = results.data
 
     for i in range(0, min(len(routes), cmdenv.routes)):

commands/update_gui.py

@@ -262,8 +262,8 @@ def validate(self, widget):
             # It seems that stations usually pay within 25% of the
             # asking price as a buy-back price. If the user gives
             # us something out of those bounds, check with them.
-            if paying < asking * 0.75 or \
-                    paying < asking - 200:
+            if paying < int(asking * 0.72) or \
+                    paying < asking - 250:
                 widget.bell()
                 ok = mbox.askokcancel(
                         "Are you sure about that?",

corrections.py

@@ -30,6 +30,7 @@
     "EKONIR/MOREYVISION": "Morey Vision",
     "TRELLA/TITTO COLONY": "Tito Colony",
     "ORERVE/WATSON SATION": "Watson Station",
+    "MCC 467/ROB HUBBARD RING": "Ron Hubbard Ring",
 
 #ADD_STATIONS_HERE
 }
@@ -45,6 +46,15 @@
     'TERRAIN ENRICH SYS': 'Land Enrichment Systems',
     'HEL-STATIC FURNACES': 'Microbial Furnaces',
     'REACTIVE ARMOR': 'Reactive Armour',
+    'COTTON': DELETED,
+    'ALLOYS': DELETED,
+    'PLASTICS': DELETED,
+    'COMBAT STABILISERS': DELETED,
+    'CONSUMER TECH': 'Consumer Technology',
+    'DOM. APPLIANCES': 'Domestic Appliances',
+    'FRUIT AND VEGETABLES': 'Fruit And Vegetables',
+    'NON-LETHAL WPNS': 'Non-Lethal Weapons',
+
 }
 
 def correctSystem(oldName):

data/AltItemNames.csv

@@ -1,10 +1,10 @@
 unq:[email protected]_id,unq:[email protected]_id,unq:alt_name
-'Consumer Items','Consumer Tech','consumertechnology'
-'Consumer Items','Dom. Appliances','domesticappliances'
+'Consumer Items','Consumer Technology','consumertechnology'
+'Consumer Items','Domestic Appliances','domesticappliances'
 'Legal Drugs','Narcotics','basicnarcotics'
 'Machinery','Microbial Furnaces','heliostaticfurnaces'
 'Medicines','Agri-Medicines','agriculturalmedicines'
 'Technology','H.E. Suits','hazardousenvironmentsuits'
 'Technology','Land Enrichment Systems','terrainenrichmentsystems'
-'Weapons','Non-Lethal Wpns','nonlethalweapons'
+'Weapons','Non-Lethal Weapons','nonlethalweapons'
 'Weapons','Reactive Armour','reactivearmour'