Skip to content

Commit

Permalink
Speed/memory improvement for prices parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
Oliver Smith committed Feb 4, 2015
1 parent fc30fc0 commit f79a7e1
Showing 1 changed file with 118 additions and 105 deletions.
223 changes: 118 additions & 105 deletions cache.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,15 @@ def __str__(self):
)


class UnknownSystemError(BuildCacheBaseException):
"""
Raised when the file contains an unknown star name.
"""
def __init__(self, fromFile, lineNo, key):
error = 'Unrecognized SYSTEM: "{}"'.format(key)
super().__init__(fromFile, lineNo, error)


class UnknownStationError(BuildCacheBaseException):
"""
Raised when the file contains an unknown star/station name.
Expand Down Expand Up @@ -433,6 +442,17 @@ def processPrices(tdenv, priceFile, db, defaultZero):

systemByName = getSystemByNameIndex(cur)
stationByName = getStationByNameIndex(cur)
stationByName.update(
(sys, ID)
for sys, ID in corrections.stations.items()
if isinstance(ID, int)
)
sysCorrections = corrections.systems
stnCorrections = {
stn: alt
for stn, alt in corrections.stations.items()
if isinstance(alt, str)
}

itemByName = getItemByNameIndex(cur)

Expand All @@ -452,15 +472,20 @@ def processPrices(tdenv, priceFile, db, defaultZero):
warnings = 0
localAdd = 0

def ignoreOrWarn(error):
nonlocal warnings
if not ignoreUnknown:
if not ignoreUnknown:
def ignoreOrWarn(error):
raise error
if not quiet:
elif not quiet:
def ignoreOrWarn(error):
error.category = "WARNING"
print(error)
warnings += 1
warnings += 1
else:
def ignoreOrWarn(error):
warnings += 1

DEBUG0, DEBUG1 = tdenv.DEBUG0, tdenv.DEBUG1
DEBUG0("Processing prices file: {}", priceFile)

def changeStation(matches):
nonlocal facility, stationID
Expand All @@ -470,94 +495,85 @@ def changeStation(matches):
systemNameIn, stationNameIn = matches.group(1, 2)
systemName, stationName = systemNameIn.upper(), stationNameIn.upper()
corrected = False
facility = systemName + '/' + stationName

tdenv.DEBUG0("NEW STATION: {}", facility)
facility = "/".join((systemName, stationName))

# Make sure it's valid.
try:
stationID = stationByName[facility]
except KeyError:
stationID = -1

if stationID < 0:
stationID = DELETED
newID = stationByName.get(facility, -1)
DEBUG0("Selected station: {}, ID={}", facility, newID)
if newID is DELETED:
DEBUG1("DELETED Station: {}", facility)
return
if newID < 0:
if checkForOcrDerp(tdenv, systemName, stationName):
stationID = DELETED
return
corrected = True
try:
correctName = corrections.systems[systemName]
if correctName == DELETED:
tdenv.DEBUG1("DELETED: {}", systemName)
stationID = DELETED
return
systemName = correctName.upper()
except KeyError:
pass
try:
key = systemName + '/' + stationName
correctName = corrections.stations[key]
if correctName == DELETED:
tdenv.DEBUG1("DELETED: {}", key)
stationID = DELETED
return
stationName = correctName.upper()
except KeyError:
pass
facility = systemName + '/' + stationName
try:
stationID = stationByName[facility]
tdenv.DEBUG1("Renamed: {}/{} -> {}",
systemNameIn, stationNameIn,
facility
)
except KeyError:
stationID = -1
altName = sysCorrections.get(systemName, None)
if altName is DELETED:
DEBUG1("DELETED System: {}", facility)
return
if altName:
DEBUG1("SYSTEM '{}' renamed '{}'", systemName, altName)
systemName, facility = altName, "/".join((altName, stationName))

if stationID < 0 and ignoreUnknown:
try:
systemID = systemByName[systemName]
except KeyError:
pass
else:
name = tradedb.TradeDB.titleFixup(stationName)
inscur = db.cursor()
inscur.execute("""
INSERT INTO Station (
system_id, name,
ls_from_star,
blackmarket,
max_pad_size
) VALUES (
?, ?, 0, '?', '?'
)
""", [systemID, name])
stationID = inscur.lastrowid
stationByName[facility] = stationID
db.commit()
tdenv.NOTE("Added local station placeholder for {} (#{})",
facility, stationID
systemID = systemByName.get(systemName, -1)
if systemID < 0:
ignoreOrWarn(
UnknownSystemError(priceFile, lineNo, facility)
)
localAdd += 1
return

if stationID < 0:
stationID = DELETED
ignoreOrWarn(
UnknownStationError(priceFile, lineNo, facility)
altStation = stnCorrections.get(facility, None)
if altStation is DELETED:
DEBUG1("DELETED Station: {}", facility)
return
if altStation:
DEBUG1("Station '{}' renamed '{}'", facility, altStation)
stationName = altStation.upper()
facility = "/".join((systemName, stationName))

newID = stationByName.get(facility, -1)
if newID is DELETED:
DEBUG1("Renamed station DELETED: {}", facility)
return

if newID < 0:
if not ignoreUnknown:
ignoreOrWarn(
UnknownStationError(priceFile, lineNo, facility)
)
return
name = tradedb.TradeDB.titleFixup(stationName)
inscur = db.cursor()
inscur.execute("""
INSERT INTO Station (
system_id, name,
ls_from_star,
blackmarket,
max_pad_size
) VALUES (
?, ?, 0, '?', '?'
)
""", [systemID, name])
newID = inscur.lastrowid
stationByName[facility] = newID
db.commit()
tdenv.NOTE("Added local station placeholder for {} (#{})",
facility, newID
)
return
localAdd += 1

# Check for duplicates
if stationID in processedStations:
if newID in processedStations:
if corrected:
# This is probably the old entry.
stationID = DELETED
return
raise MultipleStationEntriesError(
priceFile, lineNo, facility,
processedStations[stationID]
)

stationID = newID
processedSystems.add(systemName)
processedStations[stationID] = lineNo
processedItems = {}
Expand All @@ -568,9 +584,9 @@ def changeStation(matches):
[stationID]
)

addItem, addBuy, addSell = items.append, buys.append, sells.append

def processItemLine(matches):
nonlocal processedItems
nonlocal items, buys, sells
itemName, modified = matches.group('item', 'time')

# Look up the item ID.
Expand All @@ -582,11 +598,11 @@ def processItemLine(matches):
oldName = itemName
itemName = corrections.correctItem(itemName)
if itemName == DELETED:
tdenv.DEBUG1("DELETED {}", oldName)
DEBUG1("DELETED {}", oldName)
return
try:
itemID = itemByName[itemName]
tdenv.DEBUG1("Renamed {} -> {}", oldName, itemName)
DEBUG1("Renamed {} -> {}", oldName, itemName)
except KeyError:
ignoreOrWarn(
UnknownItemError(priceFile, lineNo, itemName)
Expand All @@ -604,7 +620,7 @@ def processItemLine(matches):
sellTo, buyFrom = matches.group('sell', 'buy')
sellTo, buyFrom = int(sellTo), int(buyFrom)
demandString, stockString = matches.group('demand', 'stock')
if demandString and stockString:
if demandString:
if demandString == "?":
demandUnits, demandLevel = -1, -1
elif demandString == "-":
Expand All @@ -613,14 +629,15 @@ def processItemLine(matches):
demandUnits, demandLevel = parseSupply(
priceFile, lineNo, 'demand', demandString
)
if stockString == "?":
stockUnits, stockLevel = -1, -1
elif stockString == "-":
stockUnits, stockLevel = 0, 0
else:
stockUnits, stockLevel = parseSupply(
priceFile, lineNo, 'stock', stockString
)
if stockString:
if stockString == "?":
stockUnits, stockLevel = -1, -1
elif stockString == "-":
stockUnits, stockLevel = 0, 0
else:
stockUnits, stockLevel = parseSupply(
priceFile, lineNo, 'stock', stockString
)
else:
demandUnits, demandLevel = defaultUnits, defaultLevel
stockUnits, stockLevel = defaultUnits, defaultLevel
Expand All @@ -630,29 +647,24 @@ def processItemLine(matches):

processedItems[itemID] = lineNo

items.append([ stationID, itemID, modified ])
addItem((stationID, itemID, modified))
if sellTo > 0 and demandUnits != 0 and demandLevel != 0:
buys.append([
stationID, itemID,
sellTo, demandUnits, demandLevel,
modified
])
addBuy((
stationID, itemID,
sellTo, demandUnits, demandLevel,
modified
))
if buyFrom > 0 and stockUnits != 0 and stockLevel != 0:
sells.append([
stationID, itemID,
buyFrom, stockUnits, stockLevel,
modified
])
addSell((
stationID, itemID,
buyFrom, stockUnits, stockLevel,
modified
))

for line in priceFile:
lineNo += 1
commentPos = line.find("#")
if commentPos >= 0:
if commentPos == 0:
continue
line = line[:commentPos]
text = line.strip()

text, _, comment = line.partition('#')
text = text.strip()
if not text:
continue

Expand Down Expand Up @@ -683,6 +695,7 @@ def processItemLine(matches):
########################################
### "+ Category" lines
if text.startswith('+'):
# we now ignore these.
continue

########################################
Expand Down

0 comments on commit f79a7e1

Please sign in to comment.