Merge branch 'master' into updates

CHANGES.txt

@@ -3,9 +3,23 @@ TradeDangerous, Copyright (C) Oliver "kfsone" Smith, July 2014
 ==============================================================================
 
 [wip]
+. (kfsone) Allow a cache rebuild at the start of a mad import
+
+v6.10.0 Feb 14 2015
+. (kfsone) Major changes to the 'maddavo' plugin:
+    - The plugin now supports "merging" of Systems and Stations from
+      maddavo's site into your local db. It does not, by default, write
+      these to the ".csv" files.
+    - Removed "--opt=syscsv" and "--opt=stncsv",
+    - Added "--opt=systems" and "--opt=stations" and "--exportcsv",
+    - Assorted cleanup
+Also:
+. (kfsone) "run" with "--progress" is less silent,
 . (kfsone) Fixed 'set changed size' error in run command
 . (kfsone) Improvements to the TradeCalc api for accessing station trades
 . (kfsone) Added "--gain-per-ton" (--gpt) to run command
+. (kfsone) Added "CSVStream" class to transfers.py
+. (kfsone) Added "Progress" progress-bar class to misc/progress.py
 + 130 new systems from EDSC
 
 v6.9.2 Feb 08 2015

README.txt

@@ -542,10 +542,10 @@ IMPORT sub-command:
 
       This has also additional options:
       --option=<option> where option is one of the following:
-        buildcache:   Forces a rebuild of the cache before processing 
-                      of the .prices file.
-        syscsv:       Also download System.csv from the site.
-        stncsv:       Also download Station.csv from the site.
+        systems:      Merge maddavo's System data into local db,
+        station:      Merge maddavo's Station data into local db,
+        exportcsv:    Regenerate System and Station .csv files after
+                      merging System/Station data.
         skipdl:       Skip doing any downloads.
         force:        Process prices even if timestamps suggest 
                       there is no new data.

cache.py

@@ -188,7 +188,6 @@
     '' |
     ^[^A-Z0-9] |
     \s{2,} |
-    \S\s\S\s |
     ^OEN |
     ^MCK(EF|FE)\b |
     \bCHAN\s+DLER |
@@ -220,7 +219,8 @@
     ,\w |
     \bI?NGLY\b |
     \bAU\sL[DO0]\b |
-    (^|\s)['.-] |
+    (^|\s)['.] |
+    ^- | -$ |
     \bDREBBFL\b
     \bLEVIE |
     \bRN\b |
@@ -230,13 +230,22 @@
     \bKLU\sDZE |
     ^[DR]HN\b |
     SU\sI?RVEY\b |
-    [A-RT-Z]PORT$ |
     H[DO0]L[O0]ING |
     H[D0]LDING |
     M[DO0]HMAN[O0] |
     \bABL\b |
     \bBENNET\b |
-    \bHU8\b
+    \bHU8\b |
+    \sCITV$ |
+    \sPIT[VY]$ |
+    \bTFR |
+    IVII |
+    \BINAI$ |
+    SET[IT]''LEMEN |
+    I'L | R'I | (^|\s)'L | [^Ss]'(?=\s|$) |
+    ^I \s (?! [Ss][Oo][Ll][Aa]) |
+    \bA7\S |
+    \sH\sI?UB$
 )''', flags=re.X)
 
 
@@ -437,13 +446,14 @@ def getItemByNameIndex(cur):
 
 
 def checkForOcrDerp(tdenv, systemName, stationName):
-    if ocrDerp.search(stationName):
+    match = ocrDerp.search(stationName)
+    if match:
         tdenv.NOTE(
             "Ignoring '{}/{}' because it looks like OCR derp."
             .format(systemName, stationName)
         )
-        return True
-    return False
+        return match
+    return None
 
 
 def processPrices(tdenv, priceFile, db, defaultZero):
@@ -577,7 +587,6 @@ def changeStation(matches):
             """, [systemID, name])
             newID = inscur.lastrowid
             stationByName[facility] = newID
-            db.commit()
             tdenv.NOTE("Added local station placeholder for {} (#{})",
                     facility, newID
             )
@@ -1113,3 +1122,28 @@ def importDataFromFile(tdb, tdenv, path, pricesFh=None, reset=False):
     # If everything worked, we may need to re-build the prices file.
     if path != tdb.pricesPath:
         regeneratePricesFile(tdb, tdenv)
+
+
+def test_derp(tdb=None, tdenv=None):
+    """
+    Test whether the station names in a trade database are free of derp.
+
+    Examples:
+        import tradedb
+        tdb = tradedb.TradeDB()
+        test_derp(tdb)
+
+        python -i cache.py
+        >>> test_derp()
+    """
+    tdb = tdb or tradedb.TradeDB()
+    tdenv = tdenv or tdb.tdenv
+    matches = 0
+    for stn in tdb.stationByID.values():
+        m = checkForOcrDerp(tdenv, stn.system.dbname, stn.dbname)
+        if m:
+            print("Match", m.groups(1))
+            matches += 1
+    if not matches:
+        print("Current data is free of known derp")
+

commands/station_cmd.py

@@ -249,6 +249,7 @@ def removeStation(tdb, cmdenv, station):
 
 def checkResultAndExportStations(tdb, cmdenv, result):
     if not result:
+        cmdenv.NOTE("No changes.")
         return None
     if cmdenv.noExport:
         cmdenv.DEBUG0("no-export set, not exporting stations")

corrections.py

@@ -9,7 +9,6 @@
 systems = {
     "22 LYNCIS": "PEPPER",
     "ALANI": DELETED,
-    "ARGETLAMH": "ARGETLáMH",
     "DJALI": "HERCULIS SECTOR QD-T B3-4",
     "PANTAA CEZISA": "GEORGE PANTAZIS",
     "TAVYTERE": "ALRAI SECTOR ON-T B3-2",
@@ -29,6 +28,7 @@
     "AULIN/ALUIN ENTERPIRSE":               "Aulin Enterprise",
     "BALDUR/DUTTON STATION":                DELETED,
     "BALTAH'SINE/BALTAH''SINE STATION":     DELETED,
+    "BALTAH'SINE/BALTAH'SINE STATION":      DELETED,
     "BALTAH'SINE/BALTHAISINE STATION":      DELETED,
     "BD+65 1846/SHARGIN BEACON":            DELETED,
     "BOLG/MOXONS MOJO":                     "Moxon's Mojo",
@@ -75,6 +75,37 @@
 
 #ADD_STATIONS_HERE
     "FK5 2550/WU INDT GATEWAY":             DELETED,
+    "BRANI/ARMSTRONG STATION":              DELETED,
+    "BRANI/BRAHE HUB":                      DELETED,
+    "BRANI/DIRAC ENTERPRISE":               DELETED,
+    "BRANI/NADDODDUR TERMINAL":             DELETED,
+    "BRANI/RIDE RING":                      DELETED,
+    "BRANI/WALLACE DOCK":                   DELETED,
+    "BETA HYDRI/EOMONDSON HIGH M'''":       DELETED,
+    "BETA HYDRI/EOMONDSON HIGH M''":        DELETED,
+    "BETA HYDRI/EOMONDSON HIGH M'":         DELETED,
+    "BETA HYDRI/FRANKLIN RINGX":            DELETED,
+    "GEORGE PANTAZIS/ZAM IKA PLATFORM":     DELETED,
+    "GEORGE PANTAZIS/ZAM R PLATFORM":       DELETED,
+    "HIP 80364/STASHEFF 'LONY":             DELETED,
+    "LALANDE 4141/4A5O4D":                  DELETED,
+    "LALANDE 4141/4A5 -O4D":                DELETED,
+    "LALANDE 4141/4A5040":                  DELETED,
+    "LTT 537/G 1RSHTEIN H 1B":              DELETED,
+    "LTT 537/RI ERRHTFIN HI BR":            DELETED,
+    "MORTEN/VE .LAZQU IEZ STATION":         DELETED,
+    "NUKURU/HALLE R PO RT":                 DELETED,
+    "PANDAMONIUM/RIEDRICH PETERS RING":     DELETED,
+    "PANDAMONIUM/.RILDRICH PLTERS RING":    DELETED,
+    "PANDAMONIUM/RILDRICH PLTERS RING":     DELETED,
+    "WOLF 1481/VE .LAZQU IEZ GATEWAY":      DELETED,
+    "BALMUNG/RU IPPE .LT STATION":          DELETED,
+    "GELONG/ROERIIIG":                      "Roe Ring",
+    "VOLOWAHKU/WATI' MINE":                 DELETED,
+    "10 KAPPA PEGASI/BONOAR HUB":           "Bondar Hub",
+    "GLIESE 868/ALVARADO RING":             DELETED,
+    "GLIESE 868/FEYNMAN TERMINAL":          DELETED,
+    "TOXANDJI/TSU NENAGA ORBITAL":          DELETED,
 }
 
 categories = {

data/ShipVendor.csv

@@ -202,6 +202,17 @@ unq:[email protected]_id,unq:[email protected]_id,unq:[email protected]_id
 'G 175-42','Werber Platform','Sidewinder'
 'G 175-42','Werber Platform','Type 6'
 'G 175-42','Werber Platform','Type 7'
+'GD 140','Liwei Terminal','Adder'
+'GD 140','Liwei Terminal','Anaconda'
+'GD 140','Liwei Terminal','Asp'
+'GD 140','Liwei Terminal','Cobra'
+'GD 140','Liwei Terminal','Dropship'
+'GD 140','Liwei Terminal','Eagle'
+'GD 140','Liwei Terminal','Hauler'
+'GD 140','Liwei Terminal','Orca'
+'GD 140','Liwei Terminal','Python'
+'GD 140','Liwei Terminal','Sidewinder'
+'GD 140','Liwei Terminal','Viper'
 'GONG GU','Kelly Dock','Adder'
 'GONG GU','Kelly Dock','Asp'
 'GONG GU','Kelly Dock','Cobra'
@@ -373,6 +384,11 @@ unq:[email protected]_id,unq:[email protected]_id,unq:[email protected]_id
 'LHS 1446','Blalock Orbital','Type 6'
 'LHS 1446','Blalock Orbital','Type 9'
 'LHS 1446','Blalock Orbital','Viper'
+'LHS 1541','Haller Port','Adder'
+'LHS 1541','Haller Port','Cobra'
+'LHS 1541','Haller Port','Eagle'
+'LHS 1541','Haller Port','Sidewinder'
+'LHS 1541','Haller Port','Type 6'
 'LHS 1914','Bohm Terminal','Adder'
 'LHS 1914','Bohm Terminal','Cobra'
 'LHS 1914','Bohm Terminal','Eagle'
@@ -496,6 +512,11 @@ unq:[email protected]_id,unq:[email protected]_id,unq:[email protected]_id
 'MEINJHALARA','Fremont Port','Orca'
 'MEINJHALARA','Fremont Port','Sidewinder'
 'MEINJHALARA','Fremont Port','Viper'
+'NADUNINDA','Robson Mines','Asp'
+'NADUNINDA','Robson Mines','Eagle'
+'NADUNINDA','Robson Mines','Hauler'
+'NADUNINDA','Robson Mines','Sidewinder'
+'NADUNINDA','Robson Mines','Type 7'
 'NETO','Ising Vision','Adder'
 'NETO','Ising Vision','Anaconda'
 'NETO','Ising Vision','Asp'
@@ -688,6 +709,16 @@ unq:[email protected]_id,unq:[email protected]_id,unq:[email protected]_id
 'THIIN','Euler Orbital','Sidewinder'
 'THIIN','Euler Orbital','Type 6'
 'THIIN','Euler Orbital','Type 7'
+'TOXANDJI','Tsunenaga Orbital','Cobra'
+'TOXANDJI','Tsunenaga Orbital','Dropship'
+'TOXANDJI','Tsunenaga Orbital','Eagle'
+'TOXANDJI','Tsunenaga Orbital','Hauler'
+'TOXANDJI','Tsunenaga Orbital','Orca'
+'TOXANDJI','Tsunenaga Orbital','Python'
+'TOXANDJI','Tsunenaga Orbital','Sidewinder'
+'TOXANDJI','Tsunenaga Orbital','Type 6'
+'TOXANDJI','Tsunenaga Orbital','Type 9'
+'TOXANDJI','Tsunenaga Orbital','Viper'
 'VOLKHAB','Vernadsky Dock','Adder'
 'VOLKHAB','Vernadsky Dock','Anaconda'
 'VOLKHAB','Vernadsky Dock','Asp'
@@ -744,3 +775,5 @@ unq:[email protected]_id,unq:[email protected]_id,unq:[email protected]_id
 'ZAONCE','Ridley Scott','Sidewinder'
 'ZAONCE','Ridley Scott','Type 6'
 'ZAONCE','Ridley Scott','Type 7'
+'ZEESSZE','Nicollier Hanger','Sidewinder'
+'ZEESSZE','Nicollier Hanger','Type 9'