diff --git a/README.txt b/README.txt index 5b513ae0..295d75c0 100644 --- a/README.txt +++ b/README.txt @@ -23,6 +23,12 @@ files from other commanders to fill out your database. == CHANGE LOG ============================================================================== +v4.6.2 Oct 25/2014 +. (kfsone) Added support for self-correcting star/station name changes, +. (kfsone) Added name corrections for maddavo's current TradeDangerous.prices, +. (kfsone) Assorted minor API changes, +. (kfsone) Minor startup optimization pass + v4.6.1 Oct 25/2014 . (kfsone) Added "--supply" (-S) which shows supply (demand and stock) fields when editing, . (kfsone) Added "--timestamps" (-T) which shows timestamp field when editing, diff --git a/buildcache.py b/buildcache.py index 78fd1f34..edae83cb 100644 --- a/buildcache.py +++ b/buildcache.py @@ -26,6 +26,7 @@ from pathlib import Path from collections import namedtuple from tradeexcept import TradeException +from data import corrections # Find the non-comment part of a string noCommentRe = re.compile(r'^\s*(?P(?:[^\\#]|\\.)*)\s*(#|$)') @@ -196,9 +197,10 @@ def genSQLFromPriceLines(priceFile, db, defaultZero, debug=0): matches = systemStationRe.match(text) if matches: # Change which station we're at - stationName = "%s/%s" % (matches.group(1).upper(), matches.group(2)) - stationID, categoryID, uiOrder = systemByName[stationName], None, 0 - if debug > 1: print("NEW STATION: {}".format(stationName)) + systemName, stationName = (corrections.correct(matches.group(1)), corrections.correct(matches.group(2))) + facility = systemName.upper() + '/' + stationName + stationID, categoryID, uiOrder = systemByName[facility], None, 0 + if debug > 1: print("NEW STATION: {}".format(facility)) continue if not stationID: print("Expecting: '@ SYSTEM / Station'.") @@ -369,6 +371,7 @@ def commandLineBuildCache(): dbFilename = TradeDB.defaultDB sqlFilename = TradeDB.defaultSQL pricesFilename = TradeDB.defaultPrices + importTables = TradeDB.defaultTables # Check command line for -w/--debug inputs. import argparse @@ -396,7 +399,7 @@ def commandLineBuildCache(): if not pricesPath.exists(): print("Prices file '{}' does not exist.".format(args.prices)) - buildCache(dbPath, sqlPath, pricesPath, args.debug) + buildCache(dbPath, sqlPath, pricesPath, importTables, debug=args.debug) if __name__ == '__main__': diff --git a/data/corrections.py b/data/corrections.py new file mode 100644 index 00000000..9b1f3061 --- /dev/null +++ b/data/corrections.py @@ -0,0 +1,14 @@ +# Provides an interface for correcting star/station names that +# have changed in recent versions. + +corrections = { + 'LOUIS DE LACAILLE PROSPECT': 'Lacaille Prospect', + 'HOPKINS HANGAR': 'Hopkins Hanger', +} + +def correct(oldName): + try: + return corrections[oldName.upper()] + except KeyError: + return oldName + diff --git a/trade.py b/trade.py index 599c1482..11086deb 100755 --- a/trade.py +++ b/trade.py @@ -62,8 +62,6 @@ from tradedb import TradeDB, AmbiguityError from tradecalc import Route, TradeCalc, localedNo -tdb = None - ###################################################################### # Helpers @@ -272,7 +270,7 @@ def run(self, route, credits): ###################################################################### # "run" command functionality. -def parseAvoids(args): +def parseAvoids(tdb, args): """ Process a list of avoidances. """ @@ -330,7 +328,7 @@ def parseAvoids(args): )) -def parseVias(args): +def parseVias(tdb, args): """ Process a list of station names and build them into a list of waypoints for the route. @@ -346,7 +344,7 @@ def parseVias(args): viaStations.add(station) -def processRunArguments(args): +def processRunArguments(tdb, args): """ Process arguments to the 'run' option. """ @@ -384,9 +382,9 @@ def processRunArguments(args): raise CommandLineError("More than one hop required to use same from/to destination") if args.avoid: - parseAvoids(args) + parseAvoids(tdb, args) if args.via: - parseVias(args) + parseVias(tdb, args) unspecifiedHops = args.hops + (0 if originStation else 1) - (1 if finalStation else 0) if len(viaStations) > unspecifiedHops: @@ -423,10 +421,12 @@ def processRunArguments(args): raise CommandLineError("Requested unique trip with more hops than there are stations...") if args.unique: if ((originStation and originStation == finalStation) or - (originStation and originStation in viaStations) or + (originStation and originStation in viaStations) or (finalStation and finalStation in viaStations)): raise CommandLineError("from/to/via repeat conflicts with --unique") + tdb.loadTrades() + if originStation and originStation.itemCount == 0: raise NoDataError("Start station {} doesn't have any price data.".format(originStation.name())) if finalStation and finalStation.itemCount == 0: @@ -443,17 +443,15 @@ def processRunArguments(args): args.mfd = None -def runCommand(args): +def runCommand(tdb, args): """ Calculate trade runs. """ - global tdb - if args.debug: print("# 'run' mode") if tdb.tradingCount == 0: raise NoDataError("Database does not contain any profitable trades.") - processRunArguments(args) + processRunArguments(tdb, args) startCr = args.credits - args.insurance routes = [ @@ -511,7 +509,7 @@ def runCommand(args): routes = [ route for route in routes if viaStations & set(route.route[viaStartPos:]) ] if not routes: - print("No routes matched your critera, or price data for that route is missing.") + print("No profitable trades matched your critera, or price data along the route is missing.") return routes.sort() @@ -562,7 +560,7 @@ def getEditorPaths(args, editorName, envVar, windowsFolders, winExe, nixExe): raise CommandLineError("ERROR: Unable to locate {} editor.\nEither specify the path to your editor with --editor or set the {} environment variable to point to it.".format(editorName, envVar)) -def editUpdate(args, stationID): +def editUpdate(tdb, args, stationID): """ Dump the price data for a specific station to a file and launch the user's text editor to let them make changes @@ -671,7 +669,7 @@ def editUpdate(args, stationID): if absoluteFilename: tmpPath.unlink() -def updateCommand(args): +def updateCommand(tdb, args): """ Allow the user to update the prices database. """ @@ -684,7 +682,7 @@ def updateCommand(args): if args._editing: # User specified one of the options to use an editor. - return editUpdate(args, stationID) + return editUpdate(tdb, args, stationID) if args.debug: print('# guided "update" mode station:{}'.format(args.station)) @@ -694,7 +692,7 @@ def updateCommand(args): ###################################################################### # -def lookupSystem(name, intent): +def lookupSystemByNameOrStation(tdb, name, intent): """ Look up a name using either a system or station name. """ @@ -708,7 +706,7 @@ def lookupSystem(name, intent): raise CommandLineError("Unknown {} system/station, '{}'".format(intent, name)) -def distanceAlongPill(sc, percent): +def distanceAlongPill(tdb, sc, percent): """ Estimate a distance along the Pill using 2 reference systems """ @@ -725,48 +723,53 @@ def distanceAlongPill(sc, percent): return dotProduct / length -def localCommand(args): + +def localCommand(tdb, args): """ Local systems """ - srcSystem = lookupSystem(args.system, 'system') + srcSystem = lookupSystemByNameOrStation(tdb, args.system, 'system') if args.ship: ship = tdb.lookupShip(args.ship) args.ship = ship if args.ly is None: args.ly = (ship.maxLyFull if args.full else ship.maxLyEmpty) ly = args.ly or tdb.maxSystemLinkLy + lySq = ly ** 2 + + tdb.buildLinks() printHeading("Local systems to {} within {} ly.".format(srcSystem.name(), ly)) distances = { } - for (destSys, destDist) in srcSystem.links.items(): + for (destSys, destDistSq) in srcSystem.links.items(): if args.debug: print("Checking {} dist={:5.2f}".format(destSys.str(), destDist)) - if destDist > ly: + if destDist > lySq: continue - distances[destSys] = destDist + distances[destSys] = math.sqrt(destDistSq) for (system, dist) in sorted(distances.items(), key=lambda x: x[1]): pillLength = "" if args.pill or args.percent: pillLengthFormat = " [{:4.0f}%]" if args.percent else " [{:5.1f}]" - pillLength = pillLengthFormat.format(distanceAlongPill(system, args.percent)) + pillLength = pillLengthFormat.format(distanceAlongPill(tdb, system, args.percent)) print("{:5.2f}{} {}".format(dist, pillLength, system.str())) if args.detail: for (station) in system.stations: stationDistance = " {} ls".format(station.lsFromStar) if station.lsFromStar > 0 else "" print("\t<{}>{}".format(station.str(), stationDistance)) -def navCommand(args): + +def navCommand(tdb, args): """ Give player directions A->B """ - srcSystem = lookupSystem(args.start, 'start') - dstSystem = lookupSystem(args.end, 'end') + srcSystem = lookupSystemByNameOrStation(tdb, args.start, 'start') + dstSystem = lookupSystemByNameOrStation(tdb, args.end, 'end') avoiding = [] if args.ship: @@ -774,6 +777,7 @@ def navCommand(args): args.ship = ship if args.maxLyPer is None: args.maxLyPer = (ship.maxLyFull if args.full else ship.maxLyEmpty) maxLyPer = args.maxLyPer or tdb.maxSystemLinkLy + maxLyPerSq = maxLyPer ** 2 if args.debug: print("# Route from {} to {} with max {} ly per jump.".format(srcSystem.name(), dstSystem.name(), maxLyPer)) @@ -781,16 +785,20 @@ def navCommand(args): openList = { srcSystem: 0.0 } distances = { srcSystem: [ 0.0, None ] } + tdb.buildLinks() + # As long as the open list is not empty, keep iterating. while openList and not dstSystem in distances: # Expand the search domain by one jump; grab the list of # nodes that are this many hops out and then clear the list. openNodes, openList = openList, {} - for (node, startDist) in openNodes.items(): - for (destSys, destDist) in node.links.items(): - if destDist > maxLyPer: + for (node, startDistSq) in openNodes.items(): + startDist = math.sqrt(startDistSq) + for (destSys, destDistSq) in node.links.items(): + if destDistSq > maxLyPerSq: continue + destDist = math.sqrt(destDistSq) dist = startDist + destDist # If we already have a shorter path, do nothing try: @@ -851,13 +859,11 @@ def present(action, system): ###################################################################### # functionality for the "cleanup" command -def cleanupCommand(args): +def cleanupCommand(tdb, args): """ Perform maintenance on the database. """ - global tdb - if args.minutes <= 0: raise CommandLineError("Invalid --minutes specification.") @@ -921,7 +927,7 @@ def cleanupCommand(args): def main(): - global args, tdb + global args parser = argparse.ArgumentParser(description='Trade run calculator', add_help=False, epilog='For help on a specific command, use the command followed by -h.') parser.set_defaults(_editing=False) @@ -1045,11 +1051,11 @@ def main(): os.chdir(str(exePath)) # load the database - tdb = TradeDB(debug=args.debug, dbFilename=args.db) + tdb = TradeDB(debug=args.debug, dbFilename=args.db, buildLinks=False, includeTrades=False) # run the commands commandFunction = args.proc - return commandFunction(args) + return commandFunction(tdb, args) ###################################################################### diff --git a/tradedb.py b/tradedb.py index 632a40a4..571ce620 100644 --- a/tradedb.py +++ b/tradedb.py @@ -52,6 +52,7 @@ class SystemNotStationError(TradeException): """ pass + ###################################################################### @@ -68,11 +69,6 @@ def __init__(self, ID, dbname, posX, posY, posZ): self.stations = [] - @staticmethod - def linkSystems(lhs, rhs, distSq): - lhs.links[rhs] = rhs.links[lhs] = math.sqrt(distSq) - - def addStation(self, station): if not station in self.stations: self.stations.append(station) @@ -106,17 +102,6 @@ def __init__(self, ID, system, dbname, lsFromStar, itemCount): system.addStation(self) - def addTrade(self, dest, trade): - """ - Add an entry reflecting that an item can be bought at this - station and sold for a gain at another. - """ - # TODO: Something smarter. - if not dest in self.tradingWith: - self.tradingWith[dest] = [] - self.tradingWith[dest].append(trade) - - def getDestinations(self, maxJumps=None, maxLyPer=None, avoiding=None): """ Gets a list of the Station destinations that can be reached @@ -126,6 +111,7 @@ def getDestinations(self, maxJumps=None, maxLyPer=None, avoiding=None): avoiding = avoiding or [] maxJumps = maxJumps or sys.maxsize maxLyPer = maxLyPer or float("inf") + maxLyPerSq = maxLyPer ** 2 # The open list is the list of nodes we should consider next for # potential destinations. @@ -134,13 +120,13 @@ def getDestinations(self, maxJumps=None, maxLyPer=None, avoiding=None): # The closed list is the list of nodes we've already been to (so # that we don't create loops A->B->C->A->B->C->...) - Node = namedtuple('Node', [ 'system', 'via', 'distLy' ]) + Node = namedtuple('Node', [ 'system', 'via', 'distLySq' ]) openList = [ Node(self.system, [], 0) ] - pathList = { system.ID: Node(system, None, 0.0) + pathList = { system.ID: Node(system, None, -1.0) # include avoids so we only have # to consult one place for exclusions - for system in avoiding + [ self ] + for system in avoiding # the avoid list may contain stations, # which affects destinations but not vias if isinstance(system, System) } @@ -156,18 +142,18 @@ def getDestinations(self, maxJumps=None, maxLyPer=None, avoiding=None): jumps += 1 for node in ring: - for (destSys, destDist) in node.system.links.items(): - if destDist > maxLyPer: continue - dist = node.distLy + destDist + for (destSys, destDistSq) in node.system.links.items(): + if destDistSq > maxLyPerSq: continue + distSq = node.distLySq + destDistSq # If we already have a shorter path, do nothing try: - if dist >= pathList[destSys.ID].distLy: continue + if distSq >= pathList[destSys.ID].distLySq: continue except KeyError: pass # Add to the path list - pathList[destSys.ID] = Node(destSys, node.via, dist) + pathList[destSys.ID] = Node(destSys, node.via, distSq) # Add to the open list but also include node to the via # list so that it serves as the via list for all next-hops. - openList += [ Node(destSys, node.via + [destSys], dist) ] + openList += [ Node(destSys, node.via + [destSys], distSq) ] Destination = namedtuple('Destination', [ 'system', 'station', 'via', 'distLy' ]) @@ -183,10 +169,10 @@ def getDestinations(self, maxJumps=None, maxLyPer=None, avoiding=None): avoidStations = [ station for station in avoiding if isinstance(station, Station) ] epsilon = sys.float_info.epsilon for node in pathList.values(): - if node.distLy > epsilon: # Values indistinguishable from zero are avoidances + if node.distLySq >= 0.0: # Values indistinguishable from zero are avoidances for station in node.system.stations: if not station in avoidStations: - destStations += [ Destination(node.system, station, [self.system] + node.via + [station.system], node.distLy) ] + destStations += [ Destination(node.system, station, [self.system] + node.via + [station.system], math.sqrt(node.distLySq)) ] return destStations @@ -334,7 +320,7 @@ class TradeDB(object): # File containing text description of prices defaultPrices = './data/TradeDangerous.prices' # array containing standard tables, csvfilename and tablename - # WARNING: order is important because of dependencys! + # WARNING: order is important because of dependencies! defaultTables = [ [ './data/Added.csv', 'Added' ], [ './data/System.csv', 'System' ], @@ -349,7 +335,7 @@ class TradeDB(object): ] - def __init__(self, dbFilename=None, sqlFilename=None, pricesFilename=None, debug=0, maxSystemLinkLy=None): + def __init__(self, dbFilename=None, sqlFilename=None, pricesFilename=None, debug=0, maxSystemLinkLy=None, buildLinks=True, includeTrades=True): self.dbPath = Path(dbFilename or TradeDB.defaultDB) self.dbURI = str(self.dbPath) self.sqlPath = Path(sqlFilename or TradeDB.defaultSQL) @@ -357,10 +343,12 @@ def __init__(self, dbFilename=None, sqlFilename=None, pricesFilename=None, debug self.importTables = TradeDB.defaultTables self.debug = debug self.conn = None + self.numLinks = None + self.tradingCount = None self.reloadCache() - self.load(maxSystemLinkLy=maxSystemLinkLy) + self.load(maxSystemLinkLy=maxSystemLinkLy, buildLinks=buildLinks, includeTrades=includeTrades) ############################################################ @@ -457,7 +445,7 @@ def _loadSystems(self): if self.debug > 1: print("# Loaded %d Systems" % len(systemByID)) - def buildLinks(self, longestJumpLy): + def buildLinks(self): """ Populate the list of reachable systems for every star system. @@ -466,20 +454,20 @@ def buildLinks(self, longestJumpLy): to be "links". """ - longestJumpSq = longestJumpLy ** 2 # So we don't have to sqrt every distance + longestJumpSq = self.maxSystemLinkLy ** 2 # So we don't have to sqrt every distance # Generate a series of symmetric pairs (A->B, A->C, A->D, B->C, B->D, C->D) # so we only calculate each distance once, and then add a link each way. # (A->B distance populates A->B and B->A, etc) - numLinks = 0 + self.numLinks = 0 for (lhs, rhs) in itertools.combinations(self.systemByID.values(), 2): dX, dY, dZ = rhs.posX - lhs.posX, rhs.posY - lhs.posY, rhs.posZ - lhs.posZ distSq = (dX * dX) + (dY * dY) + (dZ * dZ) if distSq <= longestJumpSq: - System.linkSystems(lhs, rhs, distSq) - numLinks += 1 + lhs.links[rhs] = rhs.links[lhs] = distSq + self.numLinks += 1 - if self.debug > 2: print("# Number of links between systems: %d" % numLinks) + if self.debug > 2: print("# Number of links between systems: %d" % self.numLinks) def lookupSystem(self, key): @@ -710,8 +698,15 @@ def loadTrades(self): lists in descending order of profit (highest profit first) """ - # I could make a view that does this, but then it makes it fiddly to - # port this to another database that perhaps doesn't support views. + if self.numLinks is None: + self.buildLinks() + + # NOTE: Overconsumption. + # We currently fetch ALL possible trades with no regard for reachability; + # as the database grows this will become problematic and we should switch + # to some form of lazy load - that is, given a star, load all potential + # trades it has within a given ly range (based on a multiple of max-ly and + # max jumps). stmt = """ SELECT src.station_id, dst.station_id , src.item_id @@ -729,15 +724,24 @@ def loadTrades(self): AND dst.demand_level != 0 AND src.ui_order > 0 AND dst.ui_order > 0 - ORDER BY profit DESC + ORDER BY src.station_id, dst.station_id, profit DESC """ self.cur.execute(stmt) stations, items = self.stationByID, self.itemByID self.tradingCount = 0 + + prevSrcStnID, prevDstStnID = None, None + srcStn, dstStn = None, None + tradingWith = None + for (srcStnID, dstStnID, itemID, srcCostCr, profitCr, stock, stockLevel, demand, demandLevel, srcAge, dstAge) in self.cur: - self.tradingCount += 1 - srcStn, dstStn, item = stations[srcStnID], stations[dstStnID], items[itemID] - srcStn.addTrade(dstStn, Trade(item, itemID, srcCostCr, profitCr, stock, stockLevel, demand, demandLevel, srcAge, dstAge)) + if srcStnID != prevSrcStnID: + srcStn, prevSrcStnID, prevDstStnID = stations[srcStnID], srcStnID, None + if dstStnID != prevDstStnID: + dstStn, prevDstStnID = stations[dstStnID], dstStnID + tradingWith = srcStn.tradingWith[dstStn] = [] + self.tradingCount += 1 + tradingWith.append(Trade(items[itemID], itemID, srcCostCr, profitCr, stock, stockLevel, demand, demandLevel, srcAge, dstAge)) def getTrades(self, src, dst): @@ -749,7 +753,7 @@ def getTrades(self, src, dst): return srcStn.tradingWith[dstStn] - def load(self, dbFilename=None, maxSystemLinkLy=None): + def load(self, dbFilename=None, maxSystemLinkLy=None, buildLinks=True, includeTrades=True): """ Populate/re-populate this instance of TradeDB with data. WARNING: This will orphan existing records you have @@ -783,9 +787,11 @@ def load(self, dbFilename=None, maxSystemLinkLy=None): self.maxSystemLinkLy = maxSystemLinkLy if self.debug > 2: print("# Max ship jump distance: %s @ %f" % (longestJumper.name(), self.maxSystemLinkLy)) - self.buildLinks(self.maxSystemLinkLy) + if buildLinks: + self.buildLinks() - self.loadTrades() + if includeTrades: + self.loadTrades() # In debug mode, check that everything looks sane. if self.debug: