diff --git a/data/dbdef.sql b/data/dbdef.sql deleted file mode 100644 index 0144af25..00000000 --- a/data/dbdef.sql +++ /dev/null @@ -1,154 +0,0 @@ --- Source for the TradeDangerous database. - --- I'm using foreign keys for referential integrity. -PRAGMA foreign_keys = ON; - --- Star systems -CREATE TABLE - System - ( - system_id INTEGER PRIMARY KEY AUTOINCREMENT, - name VARCHAR(40) COLLATE nocase, - pos_x DOUBLE NOT NULL, - pos_y DOUBLE NOT NULL, - pos_z DOUBLE NOT NULL, - modified DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, - - UNIQUE (name) - ) -; -CREATE INDEX systems_position ON System (pos_x, pos_y, pos_z); - --- Stations within systems -CREATE TABLE - Station - ( - station_id INTEGER PRIMARY KEY AUTOINCREMENT, - name VARCHAR(40) COLLATE nocase, - system_id INTEGER NOT NULL, - ls_from_star DOUBLE NOT NULL, - - UNIQUE (name), - - FOREIGN KEY (system_id) REFERENCES System(system_id) - ON UPDATE CASCADE - ON DELETE CASCADE - ) -; -CREATE INDEX station_systems ON Station (system_id); - --- Ships -CREATE TABLE - Ship - ( - ship_id INTEGER PRIMARY KEY AUTOINCREMENT, - name VARCHAR(40) COLLATE nocase, - capacity INTEGER NOT NULL, - mass INTEGER NOT NULL, - drive_rating DOUBLE NOT NULL, - max_ly_empty DOUBLE NOT NULL, - max_ly_full DOUBLE NOT NULL, - max_speed INTEGER NOT NULL, - boost_speed INTEGER NOT NULL, - - UNIQUE (name) - ) -; - --- Where ships can be bought -CREATE TABLE - ShipVendor - ( - ship_id INTEGER NOT NULL, - station_id INTEGER NOT NULL, - cost INTEGER, - - PRIMARY KEY (ship_id, station_id), - - FOREIGN KEY (ship_id) REFERENCES Ship(ship_id) - ON UPDATE CASCADE - ON DELETE CASCADE, - FOREIGN KEY (station_id) REFERENCES Station(station_id) - ON UPDATE CASCADE - ON DELETE CASCADE - ) WITHOUT ROWID -; - -CREATE TABLE - Upgrade - ( - upgrade_id INTEGER PRIMARY KEY AUTOINCREMENT, - name VARCHAR(40) COLLATE nocase, - weight NUMBER NOT NULL, - - UNIQUE (name) - ) -; - -CREATE TABLE - UpgradeVendor - ( - upgrade_id INTEGER NOT NULL, - station_id INTEGER NOT NULL, - cost INTEGER, - - PRIMARY KEY (upgrade_id, station_id), - - FOREIGN KEY (upgrade_id) REFERENCES Upgrade(upgrade_id) - ON UPDATE CASCADE - ON DELETE CASCADE, - FOREIGN KEY (station_id) REFERENCES Station(station_id) - ON UPDATE CASCADE - ON DELETE CASCADE - ) WITHOUT ROWID -; - --- Trade items are divided up into categories -CREATE TABLE - Category - ( - category_id INTEGER PRIMARY KEY AUTOINCREMENT, - name VARCHAR(40) COLLATE nocase, - - UNIQUE (name) - ) -; - --- Tradeable items -CREATE TABLE - Item - ( - item_id INTEGER PRIMARY KEY AUTOINCREMENT, - name VARCHAR(40) COLLATE nocase, - category_id INTEGER NOT NULL, - - UNIQUE (category_id, name), - - FOREIGN KEY (category_id) REFERENCES Category(category_id) - ON UPDATE CASCADE - ON DELETE CASCADE - ) -; - -CREATE TABLE - Price - ( - item_id INTEGER NOT NULL, - station_id INTEGER NOT NULL, - ui_order INTEGER NOT NULL DEFAULT 0, - -- how many credits will the station pay for this item? - sell_to INTEGER NOT NULL, - -- how many credits must you pay to buy at this station? - buy_from INTEGER NOT NULL DEFAULT 0, - modified DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, - - PRIMARY KEY (item_id, station_id), - - FOREIGN KEY (item_id) REFERENCES Item(item_id) - ON UPDATE CASCADE - ON DELETE CASCADE, - FOREIGN KEY (station_id) REFERENCES Station(station_id) - ON UPDATE CASCADE - ON DELETE CASCADE - ) WITHOUT ROWID -; diff --git a/misc/import-accdb-to-sq3.py b/misc/import-accdb-to-sq3.py deleted file mode 100644 index 21ac695d..00000000 --- a/misc/import-accdb-to-sq3.py +++ /dev/null @@ -1,256 +0,0 @@ -#! /usr/bin/env python -###################################################################### -# Copyright (C) Oliver 'kfsone' Smith 2014 : -# You are free to use, redistribute, or even print and eat a copy of -# this software so long as you include this copyright notice. -# I guarantee there is at least one bug neither of us knew about. -###################################################################### -# CAUTION: NOT INTENDED FOR GENERAL END-USER OPERATION. -# If you don't know what this script is for, you can safely ignore it. -###################################################################### -# TradeDangerous :: Misc :: Legacy data import. -# -# Bootstrap a new SQLite3 database from python files and an existing -# Microsoft Access .ACCDB database. -# -# Note: This is NOT intended to be user friendly. If you don't know what this -# script is for, then you can safely ignore it. - -# Main imports. -import sys, os, re - -# We'll need a list of star systems. -import dataseed.stars -import dataseed.ships -from tradedb import * - -# Filenames/URIs -dbDef = "dataseed/dbdef.sql" -inDB = "Driver={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=.\\TradeDangerous.accdb" -outDB = TradeDB.defaultDB - -systems, systemByID = {}, {} -stations, stationByOldID = {}, {} -categories, categoryByOldID = {}, {} -items, itemByOldID = {}, {} -debug = 1 - -# We also track the maximum distance any ship can jump, -# then we use this value to constrain links between stations. -maxJumpDistanceLy = 0.0 - -###################################################################### -# Helpers - -class check_item(object): - """ - Wrapper class that allows us to beautify the output as a sort - of checklist. - - Usage: - with check_item("Step description"): - things_in_step() - """ - margin = 60 - def __init__(self, title): - self.title, self.noop = title, False - def __enter__(self): - print('- {:.<{width}}: '.format(self.title, width=self.margin - 3), end='') - return self - def __exit__(self, type, value, traceback): - if value: # Exception occurred - print("\a\rX {:.<{width}}: ERROR".format(self.title.upper(), width=self.margin - 3)) - print() - else: - print('[+]') if not self.noop else print("\bNO-OP") - -def debug_log(level, message): - if debug >= level: - print(" | {:^54} |".format(message)) - -###################################################################### -# Main - -def main(): - # Destroy the SQLite database if it already exists. - try: os.remove(outDB) - except: pass - - with check_item("Connect to MS Access DB"): - import pypyodbc - inConn = pypyodbc.connect(inDB) - inCur = inConn.cursor() - - with check_item("Connect to SQLite3 DB"): - import sqlite3 - outConn = sqlite3.connect(outDB) - outCur = outConn.cursor() - # Add a function for calculating distance between stars - outConn.create_function("calc_distance_sq", 6, TradeDB.distanceSq) - - with check_item("Apply DDL commands from '%s'" % dbDef): - # Sure, I could have written: outCur.executescript(open(dbDef).read()).commit() - # but my perl days are behind me... - ddlScript = open(dbDef).read() - outCur.executescript(ddlScript) - - with check_item("Populate `System` table"): - # Star data is stored in 'stars.py' - stmt = "INSERT INTO System (name, pos_x, pos_y, pos_z) VALUES (?, ?, ?, ?)" - for star in data.stars.stars: - outCur.execute(stmt, [ star.name, star.x, star.y, star.z ]) - systemID = int(outCur.lastrowid) - system = System(systemID, star.name, star.x, star.y, star.z) - systems[TradeDB.normalizedStr(star.name)] = system - systemByID[systemID] = system - debug_log(1, "{} star systems loaded".format(len(systems))) - - with check_item("Populate `Station` table"): - # We're going to remap the station IDs to new IDs, and we're the accessdb - # hails back to ED:Beta 1 so it doesn't distinguish between systems and - # stations, so we'll need to make the associations between stations - # and the systems they are in. - - # Systems without a station were represented by a station called 'STARNAME*', - # and we're going to want to filter those out. - fakeNameRe = re.compile(r'\*$') - - inCur.execute("SELECT ID, station, system FROM Stations ORDER BY ID") - stmt = "INSERT INTO Station (name, system_id, ls_from_star) VALUES (?, ?, 0)" - for (ID, stationName, systemName) in inCur: - if fakeNameRe.search(stationName): - continue - - system = systems[TradeDB.normalizedStr(systemName)] - - outCur.execute(stmt, [ stationName, system.ID ]) - newStationID = int(outCur.lastrowid) - oldStationID = int(ID) - - station = Station(newStationID, system, stationName) - stations[stationName] = station - stationByOldID[oldStationID] = newStationID - debug_log(1, "{} stations loaded".format(len(stations))) - - with check_item("Populate `Ship` table"): - global maxJumpDistanceLy - # I'm not entirely sure whether I really want this data in the database, - # it seems perfectly fine to maintain it as a python script. - stmt = """ - INSERT INTO Ship - ( name, capacity, mass, drive_rating, max_ly_empty, max_ly_full, max_speed, boost_speed ) - VALUES - ( ?, ?, ?, ?, ?, ?, ?, ? ) - """ - rows = [] - for ship in data.ships.ships: - assert ship.maxJump > 0 and ship.maxJumpFull > 0 - assert ship.maxJumpFull <= ship.maxJump - rows += [ [ - ship.name - , ship.capacity, ship.mass, ship.driveRating - , ship.maxJump, ship.maxJumpFull - , ship.maxSpeed, ship.boostSpeed - ] ] - maxJumpDistanceLy = max(maxJumpDistanceLy, ship.maxJump, ship.maxJumpFull) - outCur.executemany(stmt, rows) - debug_log(1, "{} ships loaded".format(len(data.ships.ships))) - debug_log(1, "Maximum distance any ship jumps: {:.2f}ly".format(maxJumpDistanceLy)) - - shipLocations = 0 - with check_item("Populate `ShipVendor` table"): - stmt = """ - INSERT INTO ShipVendor - ( ship_id, station_id, cost ) - VALUES - ( (SELECT ship_id FROM Ship WHERE Ship.name = ?), ?, ? ) - """ - rows = [] - for ship in data.ships.ships: - for stationName in ship.stations: - station = stations[TradeDB.listSearch("Station", stationName, stations)] - rows += [ [ - ship.name, station.ID, 0 # We don't have prices yet. - ] ] - outCur.executemany(stmt, rows) - shipLocations = len(rows) - debug_log(1, "{} ship locations loaded".format(shipLocations)) - - with check_item("Populate `Upgrade` table") as check: - # TODO: Populate Upgrade - check.noop = True - - with check_item("Populate `UpgradeVendor' table") as check: - # TODO: UpgradeVendor - check.noop = True - - with check_item("Populate `Category` table") as check: - # Copy from accdb and track the newly assigned ID. - inCur.execute("SELECT ID, category FROM Categories ORDER BY ID") - categoryID = 0 - stmt = "INSERT INTO Category (category_id, name) VALUES (?, ?)" - rows = [] - for (ID, categoryName) in inCur: - categoryID += 1 - rows += [ [ categoryID, categoryName ] ] - categories[TradeDB.normalizedStr(categoryName)] = categoryID - categoryByOldID[ID] = categoryID - outCur.executemany(stmt, rows) - debug_log(1, "{} item categories loaded".format(len(categories))) - - with check_item("Populate `Item` table") as check: - inCur.execute("SELECT ID, category_id, item FROM Items") - stmt = "INSERT INTO Item (item_id, category_id, name) VALUES (?, ?, ?)" - rows = [] - itemID = 0 - for (ID, category_id, itemName) in inCur: - itemID += 1 - rows += [ [ itemID, categoryByOldID[category_id], itemName ] ] - items[TradeDB.normalizedStr(itemName)] = itemID - itemByOldID[ID] = itemID - outCur.executemany(stmt, rows) - debug_log(1, "{} items loaded".format(len(items))) - - pricesCount = 0 - with check_item("Populate `Price` table") as check: - inCur.execute("SELECT station_id, item_id, sell_cr, buy_cr, ui_order FROM Prices ORDER BY item_id, station_id") - stmt = """ - INSERT INTO Price - (item_id, station_id, ui_order, sell_to, buy_from) - VALUES - (?, ?, ?, ?, ?) - """ - rows = [] - for (oldStationID, oldItemID, stnPayingCr, stnAskingCr, uiOrder) in inCur: - itemID, stationID = itemByOldID[oldItemID], stationByOldID[oldStationID] - rows += [ [ itemID, stationID, uiOrder or 0, stnPayingCr, stnAskingCr or 0 ] ] - outCur.executemany(stmt, rows) - pricesCount = len(rows) - debug_log(1, "{} prices loaded".format(pricesCount)) - - outConn.commit() - - numLinks = 0 - with check_item("Checking system links/performance") as check: - import math - outCur.execute(""" - SELECT from_system_id, to_system_id, distance_sq - FROM vLink - WHERE from_system_id < to_system_id - AND distance_sq <= ? - """, [ maxJumpDistanceLy ** 2]) - links = {} - for (lhsID, rhsID, distSq) in outCur: - # Round the distance up to the next 100th of a lightyear. - distLy = math.ceil(math.sqrt(distSq) * 100.0) / 100.0 - # Generate a 64-bit identifier that describes a link - # between two systems that can be consistent regardless - # of which way round you use them (that is, you always - # have to do the min/max the same for each 32 bits) - linkID = (min(lhsID, rhsID) << 32) | (max(lhsID, rhsID)) - links[linkID] = distLy - numLinks += 1 - debug_log(1, "Number of links calculated: {}".format(numLinks)) - -if __name__ == "__main__": - main() diff --git a/misc/import-text-to-accdb.py b/misc/import-text-to-accdb.py deleted file mode 100644 index 1e24c5c1..00000000 --- a/misc/import-text-to-accdb.py +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/env python -# TradeDangerous :: Scripts :: Importer -# TradeDangerous Copyright (C) Oliver 'kfsone' Smith 2014 : -# You are free to use, redistribute, or even print and eat a copy of this -# software so long as you include this copyright notice. I guarantee that -# there is at least one bug neither of us knew about. -# -# Tool for importing data into the TradeDangerous database from a fairly -# simple text file, 'import.txt'. -# -# The first none white-space character of a line determines the line type: -# '#' = comment, -# '*' = add star and distances, -# '@' = select station, -# '-' = select item category, -# All other lines are treated as specifying an item price: -# -# or for an item that can be bought here: -# -# Blank lines are ignored. -# -# Add star line: -# */:@ -# Examples: -# *Aulin/Enterprise:Dahan@9.6 -# *Dahan/Gateway:Aulin@9.6,Eranin@8.4ly,Hermitage@14ly,Ross1015@21.00 -# -# Category and Item names use partial matching. -# -che, -CheMiCAlS, -micals -# all match the "chemicals" category -# Don't include whitespaces, so -# dom., dom.a, dom.appl -# would match the "dom. appliances" item (but only if you have selected -# the category at the moment) - -from tradedb import * - -# Assume that we're going to allow unknown stars for pre-declarations -rejectUnknown = False - -tdb = TradeDB(r'.\TradeDangerous.accdb') - -# Fetch a list of item categories, since TradeDB doesn't load it yet. -categories = dict() -for row in tdb.fetch_all(""" - SELECT Categories.category, Items.item - FROM Categories INNER JOIN Items ON Categories.ID = Items.category_id -"""): - (cat, item) = row - if not cat in categories: - categories[cat] = [] - categories[cat].append(item) - - -def addLinks(station, links): - """ Add a list of links to nearby stars. DEPRECATED. """ - global tdb - - srcID = station.ID - for dst in links.split(','): - dst, dist = dst.strip(), 1 - m = re.match(r'(.+)\s*@\s*(\d+(\.\d+)?)(\s*ly)?', dst) - if m: - dst, dist = m.group(1), m.group(2) - try: - dstID = tdb.lookupStation(dst).ID - try: - tdb.query("INSERT INTO Links (`from`, `to`, `distLy`) VALUES (%d, %d, %s)" % (srcID, dstID, dist)).commit() - except pypyodbc.IntegrityError: - tdb.query("UPDATE Links SET distLy=%s WHERE from=%d and to=%d" % (dist, srcID, dstID)).commit() - try: - tdb.query("INSERT INTO Links (`from`, `to`, `distLy`) VALUES (%d, %d, %s)" % (dstID, srcID, dist)).commit() - except pypyodbc.IntegrityError: - tdb.query("UPDATE Links SET distLy=%s WHERE from=%d and to=%d" % (dist, dstID, srcID)).commit() - except LookupError as e: - if rejectUnknown: - raise e - print("* Unknown star system: %s" % dst) - - -def changeStation(line): - global tdb - - matches = re.match(r'\s*(.*?)\s*/\s*(.*?)(:\s*(.*?))?\s*$', line) - if matches: - # Long format: system/station:links ... - sysName, stnName, links = matches.group(1), matches.group(2), matches.group(4) - if stnName == '*': - stnName = sysName.upper().join('*') - try: - station = tdb.lookupStation(stnName) - except LookupError: - tdb.query("INSERT INTO Stations (system, station) VALUES (?, ?)", [sysName, stnName]).commit() - print("Added %s/%s" % (sysName, stnName)) - tdb.load() - station = tdb.lookupStation(stnName) - if links: - addLinks(station, links) - else: - # Short format: system/station name. - station = tdb.lookupStation(line) - - print("Station: ", station) - return station - - -def changeCategory(name): - cat = tdb.listSearch('category', name, categories) - print("Category Select: ", cat) - return cat - - -def parseItem(station, cat, line, uiOrder): - fields = line.split() - itemName, sellCr, buyCr = fields[0], int(fields[1]), int(fields[2] if len(fields) > 2 else 0) - item = tdb.listSearch('item', itemName, categories[cat]) - print("Item: ", item, sellCr, buyCr) - - stationID, itemID = int(station.ID), int(tdb.itemIDs[item]) - try: - tdb.query(""" - INSERT INTO Prices (station_id, item_id, sell_cr, buy_cr, ui_order) - VALUES (%d, %d, %d, %d, %d)""" % (stationID, itemID, sellCr, buyCr, uiOrder)).commit() - except pypyodbc.IntegrityError as e: - if int(e.value[0]) == 23000: - tdb.query(""" - UPDATE Prices SET sell_cr=%d, buy_cr=%d, ui_order=%d - WHERE station_id = %d AND item_id = %d - """ % (sellCr, buyCr, uiOrder, stationID, itemID)).commit() - else: - raise e - - -def main(): - with open('import.txt') as f: - curStation = None - curCat = None - uiOrder = 0 - for line in f: - line = line.strip() - if not line or len(line) < 1: - continue - if line[0] == '#': - if line == '#rejectUnknown': - rejectUnknown = True - if line == '#stop': - break - if line[0:5] == '#echo': - text = line[6:].strip() - print(text) - continue # comment - elif line[0] == '*' or line[0] == '@': - curStation = changeStation(line[1:]) - elif line[0] == '-': - curCat = changeCategory(line[1:]) - uiOrder = 0 - else: - if curStation is None or curCat is None: - raise ValueError("Expecting station and category before items: " + line) - uiOrder += 1 - parseItem(curStation, curCat, line, uiOrder) - -if __name__ == "__main__": - main()