Merged kfsone/tradedangerous into master

CHANGES.txt

@@ -2,6 +2,19 @@
 TradeDangerous, Copyright (C) Oliver "kfsone" Smith, July 2014
 ==============================================================================
 
+v6.6.0 Jan 08 2015
+. (kfsone) Overhaul of loading of trades and adjancent-system finding
+  - Item data is loaded as discrete sales and purchases in TradeCalc,
+  - Trades (X Selling, Y Buying) are lazily populated by TradeCalc,
+  - TradeDB now builds a "StellarGrid" rather than relying on the
+    database for locality searches; massive perf improvement for Windows
+  - Reduces memory footprint and should fix memory errors 32-bit users
+    were having.
+. (kfsone) Fixed import command barfing if you didn't specify a filename
+. (kfsone) Added a hop-no display during "run" command,
+. (kfsone) Fixed a stupid command line problem with 'station' command,
++ Data: Path O'Gen, kfsone
+
 v6.5.0 Jan 06 2015
 . (sarbian/kfsone) Added "--pad-size" option to "run" command to
         restrict results to given pad sizes.

cache.py

@@ -866,6 +866,8 @@ def buildCache(tdb, tdenv):
             processImportFile(tdenv, tempDB, Path(importName), importTable)
         except FileNotFoundError:
             tdenv.DEBUG0("WARNING: processImportFile found no {} file", importName)
+        except StopIteration:
+            tdenv.NOTE("{} exists but is empty. Remove it or add the column definition line.", importName)
 
     # Parse the prices file
     if pricesPath.exists():

commands/import_cmd.py

@@ -92,8 +92,9 @@ def run(results, cmdenv, tdb):
     tdb.reloadCache()
     tdb.close()
 
-    if re.match("^https?://", cmdenv.filename, re.IGNORECASE):
-        cmdenv.url, cmdenv.filename = cmdenv.filename, None
+    if cmdenv.filename:
+        if re.match(r"^https?://", cmdenv.filename, re.IGNORECASE):
+            cmdenv.url, cmdenv.filename = cmdenv.filename, None
 
     if cmdenv.url:
         cmdenv.filename = cmdenv.filename or "import.prices"
@@ -142,4 +143,4 @@ def run(results, cmdenv, tdb):
     cache.importDataFromFile(tdb, cmdenv, filePath, pricesFh=fh)
 
     return None
-
+

commands/local_cmd.py

@@ -13,10 +13,15 @@
 
 name='local'
 help='Calculate local systems.'
-epilog=None
+epilog="See also the 'station' sub-command."
 wantsTradeDB=True
 arguments = [
-    ParseArgument('near', help='System to measure from', type=str),
+    ParseArgument(
+            'near',
+            help='Name of the system to query from.',
+            type=str,
+            metavar='SYSTEMNAME',
+    ),
 ]
 switches = [
     ParseArgument('--ly',

commands/rares_cmd.py

@@ -1,20 +1,34 @@
 from __future__ import absolute_import, with_statement, print_function, division, unicode_literals
+from commands.commandenv import ResultRow
 from commands.exceptions import *
 from commands.parsing import MutuallyExclusiveGroup, ParseArgument
+from formatting import RowFormat, ColumnFormat
 from tradedb import TradeDB
 
 import math
 
 ######################################################################
 # Parser config
 
+# Displayed on the "trade.py --help" command list, etc.
 help='Find rares near your current local.'
+# Should match the name of the module minus the _cmd.py
 name='rares'
+# Displayed at the end of the "trade.py rares --help"
 epilog=None
+# Set to False in commands that need to operate without
+# a trade database.
 wantsTradeDB=True
+# Required parameters
 arguments = [
-    ParseArgument('near', help='Your current system.', type=str),
+    ParseArgument(
+            'near',
+            help='Your current system.',
+            type=str,
+            metavar='SYSTEMNAME',
+    ),
 ]
+# Optional parameters
 switches = [
     ParseArgument('--ly',
             help='Maximum distance to search.',
@@ -50,29 +64,76 @@
 # Perform query and populate result set
 
 def run(results, cmdenv, tdb):
-    from commands.commandenv import ResultRow
-
+    """
+    Fetch all the data needed to display the results of a "rares"
+    command. Does not actually print anything.
+
+    Command execution is broken into two steps:
+        1. cmd.run(results, cmdenv, tdb)
+            Gather all the data required but generate no output,
+        2. cmd.render(results, cmdenv, tdb)
+            Print output to the user.
+
+    This separation of concerns allows modularity; you can write
+    a command that calls another command to fetch data for you
+    and knowing it doesn't generate any output. Then you can
+    process the data and return it and let the command parser
+    decide when to turn it into output.
+
+    It also opens a future door to commands that can present
+    their data in a GUI as well as the command line by having
+    a custom render() function.
+
+    Parameters:
+        results
+            An object to be populated and returned
+        cmdenv
+            A CommandEnv object populated with the parameters
+            for the command.
+        tdb
+            A TradeDB object to query against.
+
+    Returns:
+        None
+            End execution without any output
+        results
+            Proceed to "render" with the output.
+    """
+
+    # Lookup the system we're currently in.
     start = cmdenv.nearSystem
+    # Hoist the padSize parameter for convenience
+    padSize = cmdenv.padSize
 
+    # Start to build up the results data.
     results.summary = ResultRow()
     results.summary.near = start
     results.summary.ly = cmdenv.maxLyPer
 
+    # The last step in calculating the distance between two
+    # points is to perform a square root. However, we can avoid
+    # the cost of doing this by squaring the distance we need
+    # to check and only 'rooting values that are <= to it.
     maxLySq = cmdenv.maxLyPer ** 2
 
-    padSize = cmdenv.padSize
+    # Look through the rares list.
     for rare in tdb.rareItemByID.values():
-        if padSize and not rare.station.checkPadSize(padSize):
-            continue
-        dist = start.distToSq(rare.station.system)
-        if maxLySq > 0 and dist > maxLySq:
-            continue
-
+        if padSize:     # do we care about pad size?
+            if not rare.station.checkPadSize(padSize):
+                continue
+        # Find the un-sqrt'd distance to the system.
+        distSq = start.distToSq(rare.station.system)
+        if maxLySq > 0: # do we have a limit on distance?
+            if distSq > maxLySq:
+                continue
+
+        # Create a row for this item
         row = ResultRow()
         row.rare = rare
-        row.dist = math.sqrt(dist)
+        row.dist = math.sqrt(distSq)
         results.rows.append(row)
 
+    # Was anything matched?
     if not results:
         print("No matches found.")
         return None
@@ -97,13 +158,20 @@ def run(results, cmdenv, tdb):
 ## Transform result set into output
 
 def render(results, cmdenv, tdb):
-    from formatting import RowFormat, ColumnFormat
+    """
+    If the "run" command returned a result set and we are running
+    from the command line, this function will be called to generate
+    the output of the command.
+    """
 
+    # Calculate the longest station name in our list.
     longestStnName = max(results.rows, key=lambda result: len(result.rare.station.name())).rare.station
     longestStnNameLen = len(longestStnName.name())
     longestRareName = max(results.rows, key=lambda result: len(result.rare.dbname)).rare
     longestRareNameLen = len(longestRareName.dbname)
 
+    # Use the formatting system to describe what our
+    # output rows are going to look at (see formatting.py)
     rowFmt = RowFormat()
     rowFmt.addColumn('Station', '<', longestStnNameLen,
             key=lambda row: row.rare.station.name())
@@ -126,10 +194,11 @@ def render(results, cmdenv, tdb):
                     TradeDB.padSizes[row.rare.station.maxPadSize]
     )
 
+    # Print a heading summary if the user didn't use '-q'
     if not cmdenv.quiet:
         heading, underline = rowFmt.heading()
         print(heading, underline, sep='\n')
 
+    # Print out our results.
     for row in results.rows:
         print(rowFmt.format(row))
-

commands/run_cmd.py

@@ -434,17 +434,6 @@ def validateRunArguments(tdb, cmdenv):
             raise NoDataError("No price data at origin stations.")
         cmdenv.origins = tradingOrigins
 
-    if startStn:
-        tdb.loadStationTrades([startStn.ID])
-        if stopStn and cmdenv.hops == 1 and not stopStn in startStn.tradingWith:
-            raise CommandLineError(
-                    "No profitable items found between {} and {}".format(
-                        startStn.name(), stopStn.name()))
-        if len(startStn.tradingWith) == 0:
-            raise NoDataError(
-                    "No data found for potential buyers for items from {}.".format(
-                        startStn.name()))
-
 
 ######################################################################
 
@@ -555,7 +544,10 @@ def run(results, cmdenv, tdb):
     results.summary.exception = ""
 
     for hopNo in range(numHops):
-        cmdenv.DEBUG1("Hop {}", hopNo)
+        if not cmdenv.quiet and not cmdenv.debug:
+            print("* Hop {}...".format(hopNo+1), end='\r')
+        elif cmdenv.debug:
+            cmdenv.DEBUG0("Hop {}...")
 
         restrictTo = None
         if hopNo == lastHop and stopStations:
@@ -590,6 +582,8 @@ def run(results, cmdenv, tdb):
             )
             break
         routes = newRoutes
+    if not cmdenv.quiet:
+        print("{:20}".format(" "), end='\r')
 
     if not routes:
         raise NoDataError("No profitable trades matched your critera, or price data along the route is missing.")

commands/station_cmd.py

@@ -220,7 +220,7 @@ def checkSystemAndStation(tdb, cmdenv):
 
     if cmdenv.system:
         envSysName = ' '.join(cmdenv.system).upper()
-        if envSysName != sysName:
+        if sysName and envSysName != sysName:
             raise CommandLineError(
                     "Mismatch between \"--system {}\" and "
                     "system name in station specifier "