Skip to content

Commit

Permalink
Merge branch 'master' into devel
Browse files Browse the repository at this point in the history
  • Loading branch information
@bgol
bgol committed Apr 9, 2015
2 parents 183eb1e + d82dd97 commit 76e8f02
Show file tree
Hide file tree
Showing 7 changed files with 421 additions and 222 deletions.
5 changes: 2 additions & 3 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
__pycache__
import.txt
tmp
*2.py
*.laccdb
*.pyc

data/TradeDangerous.db
data/TradeDangerous.old
data/TradeDangerous.db-journal
data/TradeDangerous.prices
data/*.stamp
Expand All @@ -16,9 +15,9 @@ data/*.txt
*.suo
*.pyperf
misc/*.csv
market/
*.orig
*.prev
*.last

wip/
pycallgraph.png
Expand Down
3 changes: 3 additions & 0 deletions CHANGES.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
TradeDangerous, Copyright (C) Oliver "kfsone" Smith, July 2014
==============================================================================

v6.17.5 Apr 08 2015
. (kfsone) Fixed problem with 'import' not updating dates

v6.17.4 Apr 06 2015
. (kfsone) Fixed issue preventing removal of items in the update UI
. (kfsone) Added "-P" alias for "--plug" option of "import" command
Expand Down
88 changes: 52 additions & 36 deletions cache.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -525,9 +525,10 @@ def processPrices(tdenv, priceFile, db, defaultZero):
processedStations = {}
processedSystems = set()
processedItems = {}
stationItemDates = {}
itemPrefix = ""
DELETED = corrections.DELETED
items, buys, sells = [], [], []
items, zeros, buys, sells = [], [], [], []

warnings = 0
localAdd = 0
Expand All @@ -552,8 +553,10 @@ def ignoreOrWarn(error):
def changeStation(matches):
nonlocal facility, stationID
nonlocal processedStations, processedItems, localAdd
nonlocal stationItemDates

### Change current station
stationItemDates = {}
systemNameIn, stationNameIn = matches.group(1, 2)
systemName, stationName = systemNameIn.upper(), stationNameIn.upper()
corrected = False
Expand Down Expand Up @@ -627,23 +630,28 @@ def changeStation(matches):
facility, newID
)
localAdd += 1

# Check for duplicates
if newID in processedStations:
if corrected:
# This is probably the old entry.
return
raise MultipleStationEntriesError(
priceFile, lineNo, facility,
processedStations[newID]
)
elif newID in processedStations:
# Check for duplicates
if not corrected:
raise MultipleStationEntriesError(
priceFile, lineNo, facility,
processedStations[newID]
)

stationID = newID
processedSystems.add(systemName)
processedStations[stationID] = lineNo
processedItems = {}

addItem, addBuy, addSell = items.append, buys.append, sells.append

cur = db.execute("""
SELECT item_id, modified
FROM StationItem
WHERE station_id = ?
""", [stationID])
stationItemDates = { ID: modified for ID, modified in cur }

addItem, addZero = items.append, zeros.append
addBuy, addSell = buys.append, sells.append
getItemID = itemByName.get

def processItemLine(matches):
Expand All @@ -667,6 +675,15 @@ def processItemLine(matches):
return
DEBUG1("Renamed {} -> {}", oldName, itemName)

if modified and modified != 'now':
lastModified = stationItemDates.get(itemID, None)
if lastModified and modified <= lastModified:
DEBUG1("Ignoring {} @ {}: {} <= {}".format(
itemName, facility,
modified, lastModified,
))
return

# Check for duplicate items within the station.
if itemID in processedItems:
raise MultipleItemEntriesError(
Expand Down Expand Up @@ -705,22 +722,22 @@ def processItemLine(matches):

processedItems[itemID] = lineNo

active = False
if sellTo > 0 and demandUnits != 0 and demandLevel != 0:
addBuy((
stationID, itemID,
sellTo, demandUnits, demandLevel,
modified
))
active = True
if buyFrom > 0 and stockUnits != 0 and stockLevel != 0:
addSell((
stationID, itemID,
buyFrom, stockUnits, stockLevel,
modified
))
active = True
addItem(((stationID, itemID, modified), active))
if sellTo == 0 and buyFrom == 0:
addZero((stationID, itemID))
else:
addItem((stationID, itemID, modified))
if sellTo > 0 and demandUnits != 0 and demandLevel != 0:
addBuy((
stationID, itemID,
sellTo, demandUnits, demandLevel,
modified
))
if buyFrom > 0 and stockUnits != 0 and stockLevel != 0:
addSell((
stationID, itemID,
buyFrom, stockUnits, stockLevel,
modified
))

for line in priceFile:
lineNo += 1
Expand Down Expand Up @@ -782,7 +799,7 @@ def processItemLine(matches):
"if you /need/ to persist them."
)

return warnings, items, buys, sells, numSys, numStn
return warnings, items, zeros, buys, sells, numSys, numStn


######################################################################
Expand All @@ -791,7 +808,7 @@ def processPricesFile(tdenv, db, pricesPath, pricesFh=None, defaultZero=False):
tdenv.DEBUG0("Processing Prices file '{}'", pricesPath)

with pricesFh or pricesPath.open('rU') as pricesFh:
warnings, items, buys, sells, numSys, numStn = processPrices(
warnings, items, zeros, buys, sells, numSys, numStn = processPrices(
tdenv, pricesFh, db, defaultZero
)

Expand All @@ -815,15 +832,14 @@ def delta(self):
DELETE FROM StationItem
WHERE station_id = ?
AND item_id = ?
""", [values[:2] for values, active in items if not active])
""", zeros)
removedItems = 0 - itemCounter.delta

db.executemany("""
DELETE FROM StationItem
WHERE station_id = ?
AND item_id = ?
AND modified < IFNULL(?, CURRENT_TIMESTAMP)
""", [values for values, active in items if active])
""", [item[:2] for item in items])
deletedItems = 0 - itemCounter.delta

insertedItems = insertedSells = insertedBuys = 0
Expand All @@ -832,7 +848,7 @@ def delta(self):
INSERT OR IGNORE INTO StationItem
(station_id, item_id, modified)
VALUES (?, ?, IFNULL(?, CURRENT_TIMESTAMP))
""", [values for values, active in items if active])
""", items)
insertedItems = itemCounter.delta
if sells:
sellCounter = Counter("StationSelling")
Expand Down Expand Up @@ -1107,7 +1123,7 @@ def buildCache(tdb, tdenv):

# Create an in-memory database to populate with our data.
tempPath = dbPath.with_suffix(".new")
backupPath = dbPath.with_suffix(".prev")
backupPath = dbPath.with_suffix(".old")

if tempPath.exists():
tempPath.unlink()
Expand Down
11 changes: 5 additions & 6 deletions commands/import_cmd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@
# Parser config

help=(
"Imports price data from a '.prices' format file. "
"Previous data for the stations included in the file "
"is removed, but other stations are not affected."
"TD data import system. On its own, this command lets you "
"merge station prices from a '.prices' file (entries in the "
"file that are older than your local data are not loaded)."
)
name='import'
epilog=(
Expand All @@ -43,7 +43,7 @@
type=str,
default=None,
),
ParseArgument('--plug',
ParseArgument('--plug', '-P',
help="Use the specified import plugin.",
type=str,
default=None,
Expand Down Expand Up @@ -162,5 +162,4 @@ def run(results, cmdenv, tdb):

cache.importDataFromFile(tdb, cmdenv, filePath, pricesFh=fh, reset=cmdenv.reset)

return None

return None
38 changes: 38 additions & 0 deletions data/ShipVendor.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,16 @@ unq:[email protected]_id,unq:[email protected]_id,unq:[email protected]_id,mod
'AULIN','Aulin Enterprise','Sidewinder','2015-03-19 17:17:50'
'AULIN','Aulin Enterprise','Type 6','2015-03-19 17:17:50'
'AULIN','Aulin Enterprise','Viper','2015-03-19 17:17:50'
'AWAWAR','Cartwright Orbital','Cobra','2015-04-07 23:25:55'
'AWAWAR','Cartwright Orbital','Dropship','2015-04-07 23:25:56'
'AWAWAR','Cartwright Orbital','Eagle','2015-04-07 23:25:57'
'AWAWAR','Cartwright Orbital','Hauler','2015-04-07 23:25:58'
'AWAWAR','Cartwright Orbital','Orca','2015-04-07 23:25:59'
'AWAWAR','Cartwright Orbital','Python','2015-04-07 23:26:00'
'AWAWAR','Cartwright Orbital','Sidewinder','2015-04-07 23:26:00'
'AWAWAR','Cartwright Orbital','Type 6','2015-04-07 23:26:01'
'AWAWAR','Cartwright Orbital','Type 9','2015-04-07 23:26:02'
'AWAWAR','Cartwright Orbital','Viper','2015-04-07 23:26:03'
'AZ CANCRI','Fisher Station','Adder','2015-03-27 02:13:45'
'AZ CANCRI','Fisher Station','Anaconda','2015-03-27 02:13:45'
'AZ CANCRI','Fisher Station','Asp','2015-03-27 02:13:45'
Expand Down Expand Up @@ -458,6 +468,12 @@ unq:[email protected]_id,unq:[email protected]_id,unq:[email protected]_id,mod
'CORICCHA','Mendeleev Gateway','Type 6','2015-03-19 18:26:17'
'CORICCHA','Mendeleev Gateway','Type 7','2015-03-19 18:26:17'
'CORICCHA','Mendeleev Gateway','Type 9','2015-03-19 18:25:39'
'DAKSHMANDI','Hevelius Enterprise','Adder','2015-04-08 00:32:28'
'DAKSHMANDI','Hevelius Enterprise','Cobra','2015-04-08 00:32:28'
'DAKSHMANDI','Hevelius Enterprise','Eagle','2015-04-08 00:32:28'
'DAKSHMANDI','Hevelius Enterprise','Hauler','2015-04-08 00:32:28'
'DAKSHMANDI','Hevelius Enterprise','Sidewinder','2015-04-08 00:32:28'
'DAKSHMANDI','Hevelius Enterprise','Viper','2015-04-08 00:32:28'
'DIMOCO','Edison Orbital','Cobra','2015-03-04 19:50:35'
'DIMOCO','Edison Orbital','Type 6','2015-03-04 19:50:35'
'DIMOCO','Edison Orbital','Type 9','2015-03-04 19:50:35'
Expand Down Expand Up @@ -715,6 +731,16 @@ unq:[email protected]_id,unq:[email protected]_id,unq:[email protected]_id,mod
'HDS 1879','Shaikh Dock','Sidewinder','2015-03-25 11:45:12'
'HDS 1879','Shaikh Dock','Type 6','2015-03-25 11:45:12'
'HDS 1879','Shaikh Dock','Type 7','2015-03-25 11:45:12'
'HIP 105368','Newcomb Dock','Adder','2015-04-09 06:50:16'
'HIP 105368','Newcomb Dock','Anaconda','2015-04-09 06:50:16'
'HIP 105368','Newcomb Dock','Asp','2015-04-09 06:50:16'
'HIP 105368','Newcomb Dock','Clipper','2015-04-09 06:50:16'
'HIP 105368','Newcomb Dock','Eagle','2015-04-09 06:50:16'
'HIP 105368','Newcomb Dock','Hauler','2015-04-09 06:50:16'
'HIP 105368','Newcomb Dock','Orca','2015-04-09 06:50:16'
'HIP 105368','Newcomb Dock','Sidewinder','2015-04-09 06:50:16'
'HIP 105368','Newcomb Dock','Viper','2015-04-09 06:50:16'
'HIP 105368','Newcomb Dock','Vulture','2015-04-09 06:50:16'
'HIP 11954','Banno Orbital','Adder','2015-03-20 23:41:24'
'HIP 11954','Banno Orbital','Cobra','2015-03-20 23:41:24'
'HIP 11954','Banno Orbital','Eagle','2015-03-20 23:41:24'
Expand Down Expand Up @@ -1681,6 +1707,13 @@ unq:[email protected]_id,unq:[email protected]_id,unq:[email protected]_id,mod
'PANDEMONIUM','Zaschka Ring','Sidewinder','2015-03-22 15:06:13'
'PANDEMONIUM','Zaschka Ring','Type 6','2015-03-22 15:06:13'
'PANDEMONIUM','Zaschka Ring','Type 9','2015-03-22 15:05:23'
'PARTHIANS','Houtman Orbital','Asp','2015-04-09 05:55:19'
'PARTHIANS','Houtman Orbital','Dropship','2015-04-09 05:55:19'
'PARTHIANS','Houtman Orbital','Eagle','2015-04-09 05:55:19'
'PARTHIANS','Houtman Orbital','Hauler','2015-04-09 05:55:19'
'PARTHIANS','Houtman Orbital','Sidewinder','2015-04-09 05:55:19'
'PARTHIANS','Houtman Orbital','Type 7','2015-04-09 05:55:19'
'PARTHIANS','Houtman Orbital','Vulture','2015-04-09 05:55:19'
'PERADJARIU','Vries Port','Adder','2015-03-04 19:50:35'
'PERADJARIU','Vries Port','Anaconda','2015-03-04 19:50:35'
'PERADJARIU','Vries Port','Cobra','2015-03-04 19:50:35'
Expand Down Expand Up @@ -1930,6 +1963,11 @@ unq:[email protected]_id,unq:[email protected]_id,unq:[email protected]_id,mod
'THIIN','Euler Orbital','Sidewinder','2015-03-04 19:50:35'
'THIIN','Euler Orbital','Type 6','2015-03-04 19:50:35'
'THIIN','Euler Orbital','Type 7','2015-03-04 19:50:35'
'THOSIAO','Delporte Market','Asp','2015-04-09 06:30:13'
'THOSIAO','Delporte Market','Eagle','2015-04-09 06:30:13'
'THOSIAO','Delporte Market','Hauler','2015-04-09 06:30:13'
'THOSIAO','Delporte Market','Sidewinder','2015-04-09 06:30:13'
'THOSIAO','Delporte Market','Type 7','2015-04-09 06:30:13'
'TOXANDJI','Tsunenaga Orbital','Cobra','2015-03-04 19:50:35'
'TOXANDJI','Tsunenaga Orbital','Dropship','2015-03-04 19:50:35'
'TOXANDJI','Tsunenaga Orbital','Eagle','2015-03-04 19:50:35'
Expand Down
Loading

0 comments on commit 76e8f02

Please sign in to comment.