Skip to content

Commit

Permalink
Cache performance: Ignore the hell out of category lines in the .pric…
Browse files Browse the repository at this point in the history 
…e file
  • Loading branch information
@kfsone
kfsone committed Jan 30, 2015
1 parent 671c65a commit e417bc2
Showing 1 changed file with 41 additions and 101 deletions.
142 changes: 41 additions & 101 deletions cache.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,6 @@
## Match the '@ SYSTEM/Station' line
systemStationRe = re.compile(r'^\@\s*(.*)/(.*)')

## Match the '+ Category' line
categoryRe = re.compile(r'^\+\s*(.*)')

## Price Line matching

# first part of any prices line is the item name and paying/asking price
Expand Down Expand Up @@ -186,13 +183,14 @@
######################################################################
# Exception classes


class BuildCacheBaseException(TradeException):
"""
Baseclass for BuildCache exceptions
Attributes:
fileName Name of file being processedStations
lineNo Line the error occurred on
error Description of the error
Baseclass for BuildCache exceptions
Attributes:
fileName Name of file being processedStations
lineNo Line the error occurred on
error Description of the error
"""
def __init__(self, fromFile, lineNo, error=None):
self.fileName = fromFile.name
Expand All @@ -210,32 +208,23 @@ def __str__(self):

class UnknownStationError(BuildCacheBaseException):
"""
Raised when the file contains an unknown star/station name.
Raised when the file contains an unknown star/station name.
"""
def __init__(self, fromFile, lineNo, key):
error = 'Unrecognized STAR/Station: "{}"'.format(key)
super().__init__(fromFile, lineNo, error)


class UnknownItemError(BuildCacheBaseException):
"""
Raised in the case of an item name that we don't know.
Attributes:
itemName Key we tried to look up.
Raised in the case of an item name that we don't know.
Attributes:
itemName Key we tried to look up.
"""
def __init__(self, fromFile, lineNo, itemName):
error = 'Unrecognized item name: "{}"'.format(itemName)
super().__init__(fromFile, lineNo, error)

class UnknownCategoryError(BuildCacheBaseException):
"""
Raised in the case of a categrory name that we don't know.
Attributes:
categoryName Key we tried to look up.
"""
def __init__(self, fromFile, lineNo, categoryName):
error = 'Unrecognized category name: "{}"'.format(categoryName)
super().__init__(fromFile, lineNo, error)


class DuplicateKeyError(BuildCacheBaseException):
"""
Expand Down Expand Up @@ -290,10 +279,10 @@ def __init__(self, fromFile, lineNo, item, prevLineNo):

class SyntaxError(BuildCacheBaseException):
"""
Raised when an invalid line is read.
Attributes:
problem The problem that occurred
text Offending text
Raised when an invalid line is read.
Attributes:
problem The problem that occurred
text Offending text
"""
def __init__(self, fromFile, lineNo, problem, text):
error = "{},\ngot: '{}'.".format(problem, text.strip())
Expand All @@ -302,7 +291,7 @@ def __init__(self, fromFile, lineNo, problem, text):

class SupplyError(BuildCacheBaseException):
"""
Raised when a supply field is incorrectly formatted.
Raised when a supply field is incorrectly formatted.
"""
def __init__(self, fromFile, lineNo, category, problem, value):
error = "Invalid {} supply value: {}. Got: {}". \
Expand All @@ -313,16 +302,18 @@ def __init__(self, fromFile, lineNo, category, problem, value):
######################################################################
# Helpers


def parseSupply(pricesFile, lineNo, category, reading):
units, level = reading[0:-1], reading[-1]
levelNo = "??LMH".find(level.upper()) -1
if levelNo < -1:
raise SupplyError(
pricesFile, lineNo, category, reading,
'Unrecognized level suffix: "{}": '
"expected one of 'L', 'M', 'H' or '?'".format(
level
))
pricesFile, lineNo, category, reading,
'Unrecognized level suffix: "{}": '
"expected one of 'L', 'M', 'H' or '?'".format(
level
)
)
try:
unitsNo = int(units)
if unitsNo < 0:
Expand All @@ -334,12 +325,13 @@ def parseSupply(pricesFile, lineNo, category, reading):
pass

raise SupplyError(
pricesFile, lineNo, category, reading,
'Unrecognized units/level value: "{}": '
"expected '-', '?', or a number followed "
"by a level (L, M, H or ?).".format(
level
))
pricesFile, lineNo, category, reading,
'Unrecognized units/level value: "{}": '
"expected '-', '?', or a number followed "
"by a level (L, M, H or ?).".format(
level
)
)


######################################################################
Expand Down Expand Up @@ -367,16 +359,9 @@ def getStationByNameIndex(cur):
return { name: ID for (ID, name) in cur }


def getCategoriesByNameIndex(cur):
""" Build category name => id index """
cur.execute("SELECT category_id, name FROM category")
return { name: ID for (ID, name) in cur }


def getItemByNameIndex(cur):
"""
Generate item name index.
unique, prefix the name with the category id.
"""
cur.execute("SELECT item_id, name FROM item")
return { name: itemID for (itemID, name) in cur }
Expand All @@ -398,15 +383,14 @@ def processPrices(tdenv, priceFile, db, defaultZero):
by reading the file handle for price lines.
"""

stationID, categoryID = None, None
stationID = None

cur = db.cursor()
ignoreUnknown = tdenv.ignoreUnknown
quiet = tdenv.quiet

systemByName = getSystemByNameIndex(cur)
stationByName = getStationByNameIndex(cur)
categoriesByName = getCategoriesByNameIndex(cur)

itemByName = getItemByNameIndex(cur)

Expand All @@ -415,7 +399,6 @@ def processPrices(tdenv, priceFile, db, defaultZero):

lineNo = 0

categoryName = None
facility = None
processedStations = {}
processedSystems = set()
Expand All @@ -438,11 +421,10 @@ def ignoreOrWarn(error):


def changeStation(matches):
nonlocal categoryID, facility, stationID
nonlocal facility, stationID
nonlocal processedStations, processedItems, localAdd

### Change current station
categoryID = None
systemNameIn, stationNameIn = matches.group(1, 2)
systemName, stationName = systemNameIn.upper(), stationNameIn.upper()
corrected = False
Expand Down Expand Up @@ -544,35 +526,6 @@ def changeStation(matches):
[stationID]
)


def changeCategory(matches):
nonlocal categoryID, categoryName

categoryName = matches.group(1)

tdenv.DEBUG1("NEW CATEGORY: {}", categoryName)

try:
categoryID = categoriesByName[categoryName]
return
except KeyError:
pass

categoryName = corrections.correctCategory(categoryName)
if categoryName == DELETED:
### TODO: Determine correct way to handle this.
raise SyntaxError("Category has been deleted.")
try:
categoryID = categoriesByName[categoryName]
tdenv.DEBUG1("Renamed: {}", categoryName)
except KeyError:
categoryID = DELETED
ignoreOrWarn(
UnknownCategoryError(priceFile, lineNo, categoryName)
)
return


def processItemLine(matches):
nonlocal processedItems
nonlocal items, buys, sells
Expand Down Expand Up @@ -602,7 +555,7 @@ def processItemLine(matches):
if itemID in processedItems:
raise MultipleItemEntriesError(
priceFile, lineNo,
"{}/{}".format(categoryName, itemName),
"{}".format(itemName),
processedItems[itemID]
)

Expand Down Expand Up @@ -686,21 +639,8 @@ def processItemLine(matches):
continue

########################################
### "+ Category" lines.
### "+ Category" lines
if text.startswith('+'):
matches = categoryRe.match(text)
if not matches:
raise SyntaxError("Unrecognized '+' line: {}".format(
text
))
changeCategory(matches)
continue
if not categoryID:
# Need a category to process any other type of line.
raise SyntaxError(priceFile, lineNo,
"Expecting '+ Category Name' line", text)

if categoryID == DELETED:
continue

########################################
Expand Down Expand Up @@ -934,9 +874,9 @@ def processImportFile(tdenv, db, importPath, tableName):
# something less likely to collide with manmade
# values when it's a compound.
keyValues = [
str(linein[col]).upper()
for col in uniqueIndexes
]
str(linein[col]).upper()
for col in uniqueIndexes
]
key = ":!:".join(keyValues)
try:
prevLineNo = uniqueIndex[key]
Expand All @@ -946,10 +886,10 @@ def processImportFile(tdenv, db, importPath, tableName):
# Make a human-readable key
key = "/".join(keyValues)
raise DuplicateKeyError(
importPath, lineNo,
"entry", key,
prevLineNo
)
importPath, lineNo,
"entry", key,
prevLineNo
)
uniqueIndex[key] = lineNo

try:
Expand Down

0 comments on commit e417bc2

Please sign in to comment.