Merged kfsone/tradedangerous into master

.gitignore

@@ -9,7 +9,6 @@ data/TradeDangerous.db
 data/TradeDangerous.db-journal
 data/TradeDangerous.prices
 *.prices
-TradeDangerous.sln
 *.suo
 *.pyperf
 misc/*.csv

CHANGES.txt

@@ -3,8 +3,15 @@ TradeDangerous, Copyright (C) Oliver "kfsone" Smith, July 2014
 ==============================================================================
 
 v6.2.1 [wip]
+. (kfsone) "buy" and "sell" --near now checks the station/system too
+. (kfsone) "buy" now shows average cost if you specify --detail (-v)
+. (kfsone) "sell" now shows average value if you specify --detail (-v)
+. (kfsone) Fixed item name matching (--avoid)
+. (kfsone) Fixed use of via in "run"
 . (kfsone) Exposed cache.regeneratePricesFile()
 . (kfsone) Call regeneratePricesFile() after calling plugin.finish()
+. (kfsone) General code cleanup (removed buildLinks and loadTrades)
+. (kfsone) Added VisualStudio pyproj (great for performance analysis)
 
 v6.2.0 Dec 11 2014
 . (kfsone) Added plugin system for "import", added a maddavo plugin.

README.txt

@@ -806,12 +806,10 @@ Construction of a wholly-default TradeDB can take a while because it loads
 a lot of data that you often probably won't need. You can speed it up by
 disabling the bulk of this with:
 
-  tdb = TradeDB(tdenv, buildLinks=False, includeTrades=False)
+  tdb = TradeDB(tdenv, loadTrades=False)
 
 If you subsequently need this data, call
 
-  tdb.buildLinks()
-or
   tdb.loadTrades()
 
 As of TD 6.0 you should need to load this data less and less. A lot of

TradeDangerous.pyproj

@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" DefaultTargets="Build">
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{d36f3ae1-f917-41e9-a3f9-61cfe02c5f0f}</ProjectGuid>
+    <ProjectHome />
+    <StartupFile>trade.py</StartupFile>
+    <SearchPath />
+    <WorkingDirectory>.</WorkingDirectory>
+    <OutputPath>.</OutputPath>
+    <ProjectTypeGuids>{888888a0-9f3d-457c-b088-3a5042f75d52}</ProjectTypeGuids>
+    <LaunchProvider>Standard Python launcher</LaunchProvider>
+    <InterpreterId />
+    <InterpreterVersion />
+    <CommandLineArguments>run -vv --ly 10.38 --empty 11.16 --cap 18 --jumps 6 --cr 131366 --from "aritimi" -s 3</CommandLineArguments>
+    <EnableNativeCodeDebugging>False</EnableNativeCodeDebugging>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)' == 'Debug'" />
+  <PropertyGroup Condition="'$(Configuration)' == 'Release'" />
+  <PropertyGroup>
+    <VisualStudioVersion Condition=" '$(VisualStudioVersion)' == '' ">10.0</VisualStudioVersion>
+    <PtvsTargetsFile>$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Python Tools\Microsoft.PythonTools.targets</PtvsTargetsFile>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="cache.py" />
+    <Compile Include="commands\buildcache_cmd.py" />
+    <Compile Include="commands\buy_cmd.py" />
+    <Compile Include="commands\commandenv.py" />
+    <Compile Include="commands\exceptions.py" />
+    <Compile Include="commands\export_cmd.py" />
+    <Compile Include="commands\import_cmd.py" />
+    <Compile Include="commands\local_cmd.py" />
+    <Compile Include="commands\nav_cmd.py" />
+    <Compile Include="commands\parsing.py" />
+    <Compile Include="commands\run_cmd.py" />
+    <Compile Include="commands\sell_cmd.py" />
+    <Compile Include="commands\test_commands.py" />
+    <Compile Include="commands\update_cmd.py" />
+    <Compile Include="commands\update_gui.py" />
+    <Compile Include="commands\__init__.py" />
+    <Compile Include="corrections.py" />
+    <Compile Include="formatting.py" />
+    <Compile Include="mfd\saitek\directoutput.py" />
+    <Compile Include="mfd\saitek\x52pro.py" />
+    <Compile Include="mfd\saitek\__init__.py" />
+    <Compile Include="mfd\__init__.py" />
+    <Compile Include="misc\coord64.py" />
+    <Compile Include="misc\diff-system-csvs.py" />
+    <Compile Include="misc\edstarquery.py" />
+    <Compile Include="mkstats.py" />
+    <Compile Include="prices.py" />
+    <Compile Include="pygui.py" />
+    <Compile Include="trade.py" />
+    <Compile Include="tradecalc.py" />
+    <Compile Include="tradedb.py" />
+    <Compile Include="tradeenv.py" />
+    <Compile Include="tradeexcept.py" />
+    <Compile Include="trading.py" />
+    <Compile Include="xfer.py" />
+  </ItemGroup>
+  <ItemGroup>
+    <Content Include="CHANGES.txt" />
+    <Content Include="data\Added.csv" />
+    <Content Include="data\AltItemNames.csv" />
+    <Content Include="data\Category.csv" />
+    <Content Include="data\Item.csv" />
+    <Content Include="data\Ship.csv" />
+    <Content Include="data\ShipVendor.csv" />
+    <Content Include="data\Station.csv" />
+    <Content Include="data\System.csv" />
+    <Content Include="data\TradeDangerous.sql" />
+    <Content Include="README.txt" />
+    <Content Include="scripts\config.sh" />
+    <Content Include="scripts\README.txt" />
+    <Content Include="scripts\tdbuyfrom" />
+    <Content Include="scripts\tdimad" />
+    <Content Include="scripts\tdloc" />
+    <Content Include="scripts\tdnav" />
+    <Content Include="scripts\tdrun" />
+    <Content Include="scripts\tdupd" />
+  </ItemGroup>
+  <ItemGroup>
+    <Folder Include="commands" />
+    <Folder Include="data" />
+    <Folder Include="mfd" />
+    <Folder Include="mfd\saitek" />
+    <Folder Include="misc" />
+    <Folder Include="scripts" />
+  </ItemGroup>
+  <Import Project="$(PtvsTargetsFile)" Condition="Exists($(PtvsTargetsFile))" />
+  <Import Project="$(MSBuildToolsPath)\Microsoft.Common.targets" Condition="!Exists($(PtvsTargetsFile))" />
+</Project>

TradeDangerous.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2013
+VisualStudioVersion = 12.0.30501.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "TradeDangerous", "TradeDangerous.pyproj", "{D36F3AE1-F917-41E9-A3F9-61CFE02C5F0F}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{D36F3AE1-F917-41E9-A3F9-61CFE02C5F0F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D36F3AE1-F917-41E9-A3F9-61CFE02C5F0F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

commands/buy_cmd.py

@@ -65,6 +65,17 @@ def run(results, cmdenv, tdb):
     item = tdb.lookupItem(cmdenv.item)
     cmdenv.DEBUG0("Looking up item {} (#{})", item.name(), item.ID)
 
+    results.summary = ResultRow()
+    results.summary.item = item
+
+    if cmdenv.detail:
+        avgPrice = tdb.query("""
+                SELECT CAST(AVG(ss.price) AS INT)
+                  FROM StationSelling AS ss
+                 WHERE ss.item_id = ?
+        """, [item.ID]).fetchone()[0]
+        results.summary.avg = avgPrice
+
     # Constraints
     tables = "StationSelling AS ss"
     constraints = [ "(item_id = {})".format(item.ID) ]
@@ -86,30 +97,29 @@ def run(results, cmdenv, tdb):
     else:
         columns.append('0')
 
-    results.summary = ResultRow()
-    results.summary.item = item
-
     nearSystem = cmdenv.nearSystem
-    distances = dict()
     if nearSystem:
         maxLy = cmdenv.maxLyPer or tdb.maxSystemLinkLy
         results.summary.near = nearSystem
         results.summary.ly = maxLy
 
         cmdenv.DEBUG0("Searching within {}ly of {}", maxLy, nearSystem.name())
+        systemRanges = {
+            system: dist
+            for system, dist in tdb.genSystemsInRange(
+                    nearSystem,
+                    maxLy,
+                    includeSelf=True,
+            )
+        }
         tables += (
-                " INNER JOIN StationLink AS sl"
-                " ON (sl.rhs_station_id = ss.station_id)"
-                )
-        columns.append('sl.dist')
-        constraints.append("(lhs_system_id = {})".format(
-                nearSystem.ID
-                ))
-        constraints.append("(dist <= {})".format(
-                maxLy
-                ))
-    else:
-        columns.append('0')
+                " INNER JOIN Station AS stn"
+                " ON (stn.station_id = ss.station_id)"
+        )
+        constraints.append("(stn.system_id IN ({}))".format(
+            ",".join(['?'] * len(systemRanges))
+        ))
+        bindValues += list(system.ID for system in systemRanges.keys())
 
     whereClause = ' AND '.join(constraints)
     stmt = """
@@ -125,12 +135,12 @@ def run(results, cmdenv, tdb):
     cur = tdb.query(stmt, bindValues)
 
     stationByID = tdb.stationByID
-    for (stationID, priceCr, stock, age, dist) in cur:
+    for (stationID, priceCr, stock, age) in cur:
         row = ResultRow()
         row.station = stationByID[stationID]
         cmdenv.DEBUG2("{} {}cr {} units", row.station.name(), priceCr, stock)
         if nearSystem:
-           row.dist = dist
+           row.dist = systemRanges[row.station.system]
         row.price = priceCr
         row.stock = stock
         row.age = age
@@ -188,3 +198,10 @@ def render(results, cmdenv, tdb):
 
     for row in results.rows:
         print(stnRowFmt.format(row))
+
+    if cmdenv.detail:
+        print("{:{lnl}} {:>10n}".format(
+                "-- Average",
+                results.summary.avg,
+                lnl=longestNameLen,
+        ))

commands/commandenv.py

@@ -163,7 +163,7 @@ def checkAvoids(self):
         # individually.
         for avoid in ','.join(avoidances).split(','):
             # Is it an item?
-            item, system, station = None, None, None
+            item, place = None, None
             try:
                 item = tdb.lookupItem(avoid)
                 avoidItems.append(item)
@@ -197,11 +197,11 @@ def checkAvoids(self):
 
     def checkVias(self):
         """ Process a list of station names and build them into a list of waypoints. """
-        viaStationNames = getattr(self, 'via', None)
-        viaStations = self.viaStations = []
+        viaPlaceNames = getattr(self, 'via', None)
+        viaPlaces = self.viaPlaces = []
         # accept [ "a", "b,c", "d" ] by joining everything and then splitting it.
-        if viaStationNames:
-            for via in ",".join(viaStationNames).split(","):
-                viaStations.append(self.tdb.lookupStation(via))
+        if viaPlaceNames:
+            for via in ",".join(viaPlaceNames).split(","):
+                viaPlaces.append(self.tdb.lookupPlace(via))
 
 

commands/nav_cmd.py

@@ -22,12 +22,6 @@
             metavar='N.NN',
             type=float,
         ),
-    ParseArgument('--aggressive',
-            help='Try more aggressively.',
-            action='count',
-            dest='aggressiveness',
-            default=0,
-        ),
     ParseArgument('--avoid',
             help='Exclude a system from the route. If you specify a station, '
                  'the system that station is in will be avoided instead.',
@@ -87,12 +81,7 @@ def getRoute(cmdenv, tdb, srcSystem, dstSystem, maxLyPer):
         cmdenv.DEBUG0("Avoiding: {}", list(avoiding))
 
     # As long as the open list is not empty, keep iterating.
-    overshoot = (cmdenv.aggressiveness * 4) + 1
-    while openList:
-        if dstSystem in distances:
-            overshoot -= 1
-            if overshoot == 0:
-                break
+    while openList and dstSystem not in distances:
 
         # Expand the search domain by one jump; grab the list of
         # nodes that are this many hops out and then clear the list.

commands/run_cmd.py

@@ -326,13 +326,18 @@ def validateRunArguments(tdb, cmdenv):
             if cmdenv.startStation == cmdenv.stopStation:
                 raise CommandLineError("Same to/from; more than one hop required.")
 
-    viaSet = cmdenv.viaSet = set(cmdenv.viaStations)
+    viaSet = cmdenv.viaSet = set(cmdenv.viaPlaces)
+    viaSystems = set()
     for place in viaSet:
-        if isinstance(place, Station) and not place.itemCount:
-            raise NoDataError(
-                        "No price data available for via station {}.".format(
-                            place.name()
-                    ))
+        if isinstance(place, Station):
+            if not place.itemCount:
+                raise NoDataError(
+                            "No price data available for via station {}.".format(
+                                place.name()
+                        ))
+            viaSystems.add(place.system)
+        else:
+            viaSystems.add(place)
 
     # How many of the hops do not have pre-determined stations. For example,
     # when the user uses "--from", they pre-determine the starting station.
@@ -343,7 +348,7 @@ def validateRunArguments(tdb, cmdenv):
         fixedRoutePoints += 1
     totalRoutePoints = cmdenv.hops + 1
     adhocRoutePoints = totalRoutePoints - fixedRoutePoints
-    if len(viaSet) > adhocRoutePoints:
+    if len(viaSystems) > adhocRoutePoints:
         raise CommandLineError(
                 "Route is not long enough for the list of '--via' "
                 "destinations you gave. Reduce the vias or try again "
@@ -376,8 +381,8 @@ def validateRunArguments(tdb, cmdenv):
     if cmdenv.unique and cmdenv.hops >= len(tdb.stationByID):
         raise CommandLineError("Requested unique trip with more hops than there are stations...")
     if cmdenv.unique:
-        startConflict = (startStn and (startStn == stop or startStn in viaSet))
-        stopConflict  = (stop and stop in viaSet)
+        startConflict = (startStn and (startStn == stopStn or startStn in viaSet))
+        stopConflict  = (stopStn and stopStn in viaSet)
         if startConflict or stopConflict:
             raise CommandLineError("from/to/via repeat conflicts with --unique")
 
@@ -408,6 +413,21 @@ def validateRunArguments(tdb, cmdenv):
             raise NoDataError("No data found for potential buyers for items from {}.".format(
                                 startStn.name()))
 
+
+######################################################################
+
+
+def filterByVia(routes, viaSet, viaStartPos):
+    matchedRoutes = list(routes)
+    for route in routes:
+        met = 0
+        for hop in route.route[viaStartPos:]:
+            if hop in viaSet or hop.system in viaSet:
+                met += 1
+        if met >= len(viaSet):
+            matchedRoutes.append(route)
+    return matchedRoutes
+
 ######################################################################
 # Perform query and populate result set
 
@@ -420,7 +440,6 @@ def run(results, cmdenv, tdb):
         raise NoDataError("Database does not contain any profitable trades.")
 
     validateRunArguments(tdb, cmdenv)
-    tdb.buildLinks()
 
     from tradecalc import TradeCalc, Route
 
@@ -483,15 +502,14 @@ def run(results, cmdenv, tdb):
             ### routes that include a, b or c. On hop 4, only include routes that
             ### already include 2 of the vias, on hop 5, require all 3.
             if viaSet:
-                routes = [ route for route in routes if viaSet & set(route.route[viaStartPos:]) ]
-        elif cmdenv.adhocHops == len(viaSet):
-            # Everywhere we're going is in the viaSet.
+                routes = filterByVia(routes, viaSet, viaStartPos)
+        elif len(viaSet) > cmdenv.adhocHops:
             restrictTo = viaSet
 
         routes = calc.getBestHops(routes, restrictTo=restrictTo)
 
     if viaSet:
-        routes = [ route for route in routes if viaSet & set(route.route[viaStartPos:]) ]
+        routes = filterByVia(routes, viaSet, viaStartPos)
 
     if not routes:
         raise NoDataError("No profitable trades matched your critera, or price data along the route is missing.")