diff --git a/.editorconfig b/.editorconfig index 73651a77..c9f05822 100644 --- a/.editorconfig +++ b/.editorconfig @@ -15,6 +15,7 @@ charset = utf-8 indent_style = space indent_size = 4 trim_trailing_whitespace = false +max_line_length = 160 # Tab indentation (no size specified) [Makefile] diff --git a/requirements/dev.txt b/requirements/dev.txt index cc8b0ef9..3bc65b51 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -7,5 +7,5 @@ flake8 importlib-metadata>=1 wheel coverage -pysimdjson~=6.0.2 +ijson rich==13.7.1 diff --git a/setup.py b/setup.py index 0f98495f..0618c06d 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ setup(name = package, version = __version__, # pylint: disable=E0602 - install_requires = ["requests", "appJar", "pysimdjson", "rich"], + install_requires = ["requests", "appJar", "ijson", "rich"], setup_requires = ["pytest-runner"], tests_require = ["pytest"], packages = ['tradedangerous', 'tradedangerous.commands', 'tradedangerous.mfd', 'tradedangerous.mfd.saitek', 'tradedangerous.misc', 'tradedangerous.plugins'], diff --git a/tests/conftest.py b/tests/conftest.py index 8293ae67..a6ff532d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,7 +32,7 @@ def pytest_addoption(parser): def pytest_collection_modifyitems(config, items): has_runslow = config.getoption("--runslow") has_runsuperslow = config.getoption("--runsuperslow") - + skip_slow = pytest.mark.skip(reason="need --runslow option to run") skip_superslow = pytest.mark.skip(reason="need --runsuperslow option to run") for item in items: diff --git a/tests/helpers.py b/tests/helpers.py index 5c8aca63..803820cc 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,7 +1,6 @@ import sys import os import re -from time import sleep from io import StringIO from pathlib import Path from contextlib import contextmanager @@ -12,7 +11,7 @@ __all__ = ['tdenv', 'captured_output', 'is_initialized'] _ROOT = os.path.abspath(os.path.dirname(__file__)) -_DEBUG=5 +_DEBUG = 5 tdenv = TradeEnv(debug=_DEBUG) @contextmanager diff --git a/tests/test_bootstrap_commands.py b/tests/test_bootstrap_commands.py index e287f8d1..79c6f446 100644 --- a/tests/test_bootstrap_commands.py +++ b/tests/test_bootstrap_commands.py @@ -8,7 +8,6 @@ def test_import_commands(self): self.assertIn('buildcache', commands.commandIndex) self.assertIn('buy', commands.commandIndex) - def test_import_buildcache_cmd(self): from tradedangerous.commands import buildcache_cmd @@ -47,7 +46,6 @@ def test_import_station_cmd(self): def test_import_update_cmd(self): from tradedangerous.commands import update_cmd - + def test_import_update_gui(self): from tradedangerous.commands import update_gui - diff --git a/tests/test_bootstrap_plugins.py b/tests/test_bootstrap_plugins.py index 310ebd40..20b64cd2 100644 --- a/tests/test_bootstrap_plugins.py +++ b/tests/test_bootstrap_plugins.py @@ -1,6 +1,6 @@ import pytest -class TestBootstrapPlugins(object): +class TestBootstrapPlugins: def test_import_traded(self): import tradedangerous as td diff --git a/tests/test_cache.py b/tests/test_cache.py index 35db5530..e817e37d 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -5,7 +5,7 @@ FakeFile = namedtuple('FakeFile', ['name']) -class TestCache(object): +class TestCache: def test_parseSupply(self): fil = FakeFile('faked-file.prices') reading = '897H' diff --git a/tests/test_commands.py b/tests/test_commands.py index 9588d98e..0d215573 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,8 +1,6 @@ #! /usr/bin/env python # pytest -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals -import sys import pytest from tradedangerous import commands from tradedangerous.commands.exceptions import UsageError, CommandLineError @@ -11,9 +9,11 @@ def cmd(): return commands.CommandIndex() + prog = "trade.py" -class TestCommands(object): + +class TestCommands: def test_dashh(self, cmd): with pytest.raises(UsageError): @@ -39,4 +39,4 @@ def test_local_validsys(self, cmd): cmd.parse([prog, 'local', 'ibootis']) def test_local_validsys_dashv(self, cmd): - cmd.parse([prog, 'local', 'ibootis', '-v']) \ No newline at end of file + cmd.parse([prog, 'local', 'ibootis', '-v']) diff --git a/tests/test_fs.py b/tests/test_fs.py index 8f04aa25..1c9d130d 100644 --- a/tests/test_fs.py +++ b/tests/test_fs.py @@ -17,7 +17,7 @@ def tdenv(): return TradeEnv() -class TestFS(object): +class TestFS: def test_copy(self, tdenv): src = fs.pathify(tdenv.templateDir, 'TradeDangerous.sql') @@ -35,7 +35,8 @@ def action(): self.result = True flag = fs.ensureflag(flagfile, action) - assert self.result == True + assert self.result is True + assert flag is not None def test_copyallfiles(self, tdenv): setup_module() diff --git a/tests/test_peek.py b/tests/test_peek.py index 855f718b..5c106600 100644 --- a/tests/test_peek.py +++ b/tests/test_peek.py @@ -1,18 +1,17 @@ -import os import random import pytest -from tradedangerous.tradedb import Trade, TradeDB, Station, System +from tradedangerous.tradedb import TradeDB, Station, System from .helpers import copy_fixtures -ORIGIN='Shinrarta Dezhra' -#tdb = None +ORIGIN = 'Shinrarta Dezhra' +# tdb = None def setup_module(): copy_fixtures() -def route_to_closest(tdb:TradeDB, origin, destinations, maxLy=15): +def route_to_closest(tdb: TradeDB, origin, destinations, maxLy=15): closest = min(destinations, key=lambda candidate: candidate.distanceTo(origin)) print("Closest:", closest.name(), closest.distanceTo(origin)) route = tdb.getRoute(origin, closest, maxLy) @@ -23,16 +22,16 @@ def route_to_closest(tdb:TradeDB, origin, destinations, maxLy=15): return route def should_skip() -> bool: - return False # os.getenv("CI") != None + return False # os.getenv("CI") != None -class TestPeek(object): +class TestPeek: """ Tests based on https://github.com/eyeonus/Trade-Dangerous/wiki/Python-Quick-Peek """ - + @pytest.mark.skipif(should_skip(), reason="does not work with CI") - def test_quick_origin(self, tdb:TradeDB): + def test_quick_origin(self, tdb: TradeDB): # Look up a particular system origin = tdb.lookupSystem(ORIGIN) @@ -70,7 +69,7 @@ def test_quick_lookupPlace(self, tdb): abe = tdb.lookupPlace("sol/hamlinc") assert isinstance(abe, Station) - + @pytest.mark.skipif(should_skip(), reason="does not work with CI") def test_quick_five(self, tdb): systemTable = tdb.systemByID.values() @@ -89,7 +88,7 @@ def test_quick_five(self, tdb): else: # Route is a list of Systems. Turn it into a list of # System names... - routeNames = [ system.name() for system, distance in route ] + routeNames = [system.name() for system, distance in route] print("Route:", routeNames) route_to_closest(tdb, origin, visitMe) @@ -106,5 +105,5 @@ def test_three_different(self, tdb): lhs = tdb.lookupSystem("lhs 380") bhr = tdb.lookupSystem("bhritzameno") - result = route_to_closest(tdb, sol, [ lhs, bhr ]) - \ No newline at end of file + route_to_closest(tdb, sol, [lhs, bhr]) + diff --git a/tests/test_tools.py b/tests/test_tools.py index 5de9e960..06e73f2f 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -1,4 +1,4 @@ -import pytest +import pytest # noqa: F401 from tradedangerous import tools from .helpers import copy_fixtures @@ -8,6 +8,6 @@ def setup_module(): copy_fixtures() -class TestTools(object): +class TestTools: def test_derp(self, capsys): tools.test_derp() diff --git a/tests/test_trade.py b/tests/test_trade.py index e7b15b0a..c4ab4b6f 100644 --- a/tests/test_trade.py +++ b/tests/test_trade.py @@ -15,39 +15,39 @@ def teardown_module(): remove_fixtures() -class TestTrade(object): +class TestTrade: def test_local_help(self): with pytest.raises(UsageError): trade([PROG, "local", "-h"]) - + def test_local_sol(self, capsys): trade([PROG, "local", "--ly=10", "--detail", "sol"]) captured = capsys.readouterr() assert "Sol 0" in captured.out assert "Ehrlich City" in captured.out - + def test_sell(self, capsys): trade([PROG, "sell", "--near=sol", "hydrogen fuel"]) captured = capsys.readouterr() assert "Sol/Mars High" in captured.out - + def test_buy(self, capsys): trade([PROG, "buy", "--near=sol", "hydrogen fuel"]) captured = capsys.readouterr() assert "Cost Units DistLy Age/days" in captured.out - + def test_export_station(self, capsys): trade([PROG, "export", "-T", "System"]) captured = capsys.readouterr() assert "NOTE: Export Table 'System'" in captured.out # TODO: check that System.csv has a fresh date - + def test_station_remove(self, capsys): # "Dekker's Yard" trade([PROG, "station", "-rm", "sol/dekkers"]) captured = capsys.readouterr() assert regex_findin(r"NOTE: Sol/Dekker's Yard \(#\d+\) removed", captured.out) - + def test_station_add(self, capsys): # "Dekker's Yard" trade([ @@ -63,20 +63,20 @@ def test_station_add(self, capsys): "sol/Dangerous Delight"]) captured = capsys.readouterr() assert regex_findin(r"NOTE: Sol/Dangerous Delight \(#\d+\) added", captured.out) - + def test_nav(self, capsys): trade([PROG, "nav", "--ly-per=50", "sol", "Shinrarta Dezhra"]) captured = capsys.readouterr() assert "System JumpLy" in captured.out assert "Shinrarta Dezhra 47" in captured.out - + def test_market(self, capsys): trade([PROG, "market", "sol/abr"]) captured = capsys.readouterr() assert regex_findin("Item[ ]{3,}Buying Selling[ ]{2,}Supply", captured.out) assert "Hydrogen Fuel" in captured.out assert regex_findin("Water[ ]{3,}323", captured.out) - + @pytest.mark.slow def test_import_edcd(self, capsys): trade([PROG, "import", "-P=edcd", "--opt=commodity"]) diff --git a/tests/test_trade_run.py b/tests/test_trade_run.py index 38c6cc17..6f435183 100644 --- a/tests/test_trade_run.py +++ b/tests/test_trade_run.py @@ -13,7 +13,7 @@ def setup_module(): copy_fixtures() -class TestTradeRun(object): +class TestTradeRun: def test_run1(self, capsys): trade([PROG, "run", "--capacity=10", "--credits=10000", "--from=sol/abr", "--jumps-per=3", "--ly-per=10.5", "--no-planet"]) captured = capsys.readouterr() diff --git a/tests/test_utils.py b/tests/test_utils.py index b04ad4ea..6b075f45 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,10 +1,10 @@ -import pytest +import pytest # noqa: F401 from tradedangerous import utils from tradedangerous import TradeEnv -class TestUtils(object): +class TestUtils: # should inherit from TestCase # TODO: Test 'von' etc. def test_titleFixup_s(self): diff --git a/tox.ini b/tox.ini index c2f17e31..57800393 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,21 @@ [tox] envlist = flake8,py37,py38,py39,py310,py311 +skipsdist = true # avoid installation +skip_missing_interpreters = true [testenv] deps = -r requirements/dev.txt -commands= +passenv = + SYSTEMDRIVE, PROGRAMFILES +platform = + linux|linux2|darwin|nt|win32|win64 +# changedir = {toxinidir}/tradedangerous # ? + +[testenv:py{37,38,39,310,311}] +deps = + -r requirements/dev.txt +commands = coverage run --source=tradedangerous -m pytest {posargs} coverage report --show-missing coverage erase @@ -15,7 +26,7 @@ setenv= [flake8] include = - tradedangerous, + tradedangerous tests exclude = .git, @@ -27,9 +38,96 @@ exclude = build, dist, data, - test-data -ignore = E266, W293 -max-line-length = 160 + test-data, + setup.py, + tradedangerous/commands/TEMPLATE.py, + tradedangerous/gui.py, + tradedangerous/submit-distances.py, +ignore = + # Not sure if these are correct. + E201, E202, E225, E131, E128, E121, E122, E123, E124, E125, E126, E127, + E226, + # Definitely unsure of this one + E251, + # These should be eliminated + E722, + E228, + E502, + W503, + W504, + # Trailing whitespace - I don't know of anything that requires that + # blank lines have indentation-matching whitespace + W291, + E221, + # Missing whitespace after : + E231, + E241, + E266, + E302, + E303, + W292, + W293, + # Blank line at eof + W391, + ; this one needs research + E731, +per-file-ignores = + tests/test_bootstrap_commands.py:F401 + tests/test_bootstrap_plugins.py:F401 + tradedangerous/commands/__init__.py:F401,E402 + tradedangerous/mfd/saitek/directoutput.py:E501,E203,E401,E722,E265 +max-line-length = 180 + + +[testenv:flake8] +deps = flake8 +commands = flake8 tradedangerous/ tests/ + + +[pylint] +disable = + too-many-lines, + too-many-instance-attributes, + too-few-public-methods, + too-many-public-methods, + invalid-name, # TODO: pythonize names + too-many-arguments, + too-many-locals, + missing-class-docstring, # TODO: docstr all the things + missing-function-docstring, # TODO: docstr all the things + missing-module-docstring, # TODO: docstr all the things + trailing-whitespace, + fixme, + trailing-newlines, + missing-final-newline, + wrong-import-order, + too-many-return-statements, + + # it would be nice if these weren't disabled + consider-using-f-string, + import-outside-toplevel, + consider-alternative-union-syntax, # sure, when it's properly supported + unused-argument, + unnecessary-pass, + broad-exception-raised, + consider-using-with, # eliminate + attribute-defined-outside-init, + too-many-branches, + too-many-statements, + global-statement, + R0801, + +max-line-length = 180 + +runtime-typing = no +load-plugins = + #pylint.extensions.bad_builtin, + pylint.extensions.overlapping_exceptions, + #pylint.extensions.check_elif, + pylint.extensions.redefined_variable_type, + pylint.extensions.typing, + +ignore = gui.py submit-distances.py [pytest] markers = diff --git a/tradedangerous/__init__.py b/tradedangerous/__init__.py index a5f60e29..fa375153 100644 --- a/tradedangerous/__init__.py +++ b/tradedangerous/__init__.py @@ -10,14 +10,14 @@ # -------------------------------------------------------------------- """ -TradeDangerous is set of powerful trading tools for Elite Dangerous, +TradeDangerous is a set of powerful trading tools for Elite Dangerous, organized around one of the most powerful trade run optimizers available. The TRO is a heavy hitter that can calculate complex routes with multiple stops while taking into account the profits you make along the route -The price data in TradeDangerous is either manually entered or crowd sourced +The price data in TradeDangerous is either manually entered or crowd-sourced from a website such as [Tromador's Trading Dangerously](http://elite.ripz.org "Tromador's Trading Dangerously"), often using a plugin such as the included eddblink. """ -from .version import * +from .version import * # noqa: F401, F403 -from .tradeenv import TradeEnv +from .tradeenv import TradeEnv # noqa: F401 diff --git a/tradedangerous/cache.py b/tradedangerous/cache.py index e4e0ad0f..9d93ff72 100644 --- a/tradedangerous/cache.py +++ b/tradedangerous/cache.py @@ -20,18 +20,27 @@ # TODO: Split prices into per-system or per-station files so that # we can tell how old data for a specific system is. -from collections import namedtuple -from pathlib import Path -from .tradeexcept import TradeException +from __future__ import annotations -from . import corrections, utils +from pathlib import Path import csv -import math import os -from . import prices import re import sqlite3 import sys +import typing + +from .tradeexcept import TradeException +from . import corrections, utils +from . import prices + + +# For mypy/pylint type checking +if typing.TYPE_CHECKING: + from typing import Any, Optional, TextIO + + from .tradeenv import TradeEnv + ###################################################################### # Regular expression patterns. Here be draegons. @@ -76,24 +85,23 @@ """ newItemPriceRe = re.compile(r""" ^ - {base_f} + {itemPriceFrag} ( \s+ # demand units and level - (?P {qtylvl_f}) + (?P {qtyLevelFrag}) \s+ # supply units and level - (?P {qtylvl_f}) + (?P {qtyLevelFrag}) # time is optional (?: \s+ - {time_f} + {timeFrag} )? )? \s* $ -""".format(base_f = itemPriceFrag, qtylvl_f = qtyLevelFrag, time_f = timeFrag), - re.IGNORECASE + re.VERBOSE) +""", re.IGNORECASE + re.VERBOSE) ###################################################################### # Exception classes @@ -108,18 +116,14 @@ class BuildCacheBaseException(TradeException): error Description of the error """ - def __init__(self, fromFile, lineNo, error = None): + def __init__(self, fromFile: Path, lineNo: int, error: str = None) -> None: self.fileName = fromFile.name self.lineNo = lineNo self.category = "ERROR" self.error = error or "UNKNOWN ERROR" - def __str__(self): - return "{}:{} {} {}".format( - self.fileName, self.lineNo, - self.category, - self.error, - ) + def __str__(self) -> str: + return f'{self.fileName}:{self.lineNo} {self.category} {self.error}' class UnknownSystemError(BuildCacheBaseException): @@ -127,9 +131,8 @@ class UnknownSystemError(BuildCacheBaseException): Raised when the file contains an unknown star name. """ - def __init__(self, fromFile, lineNo, key): - error = 'Unrecognized SYSTEM: "{}"'.format(key) - super().__init__(fromFile, lineNo, error) + def __init__(self, fromFile: Path, lineNo: int, key: str) -> None: + super().__init__(fromFile, lineNo, f'Unrecognized SYSTEM: "{key}"') class UnknownStationError(BuildCacheBaseException): @@ -137,9 +140,8 @@ class UnknownStationError(BuildCacheBaseException): Raised when the file contains an unknown star/station name. """ - def __init__(self, fromFile, lineNo, key): - error = 'Unrecognized STAR/Station: "{}"'.format(key) - super().__init__(fromFile, lineNo, error) + def __init__(self, fromFile: Path, lineNo: int, key: str) -> None: + super().__init__(fromFile, lineNo, f'Unrecognized STAR/Station: "{key}"') class UnknownItemError(BuildCacheBaseException): @@ -149,9 +151,8 @@ class UnknownItemError(BuildCacheBaseException): itemName Key we tried to look up. """ - def __init__(self, fromFile, lineNo, itemName): - error = 'Unrecognized item name: "{}"'.format(itemName) - super().__init__(fromFile, lineNo, error) + def __init__(self, fromFile: Path, lineNo: int, itemName: str) -> None: + super().__init__(fromFile, lineNo, f'Unrecognized item name: "{itemName}"') class DuplicateKeyError(BuildCacheBaseException): @@ -159,14 +160,9 @@ class DuplicateKeyError(BuildCacheBaseException): Raised when an item is being redefined. """ - def __init__(self, fromFile, lineNo, keyType, keyValue, prevLineNo): + def __init__(self, fromFile: Path, lineNo: int, keyType: str, keyValue: str, prevLineNo: int) -> None: super().__init__(fromFile, lineNo, - "Second occurrance of {keytype} \"{keyval}\", " - "previous entry at line {prev}.".format( - keytype = keyType, - keyval = keyValue, - prev = prevLineNo - )) + f'Second occurrance of {keyType} "{keyValue}", previous entry at line {prevLineNo}.') class DeletedKeyError(BuildCacheBaseException): @@ -175,11 +171,11 @@ class DeletedKeyError(BuildCacheBaseException): corrections file. """ - def __init__(self, fromFile, lineNo, keyType, keyValue): - super().__init__(fromFile, lineNo, - "{} '{}' is marked as DELETED and should not be used.".format( - keyType, keyValue - )) + def __init__(self, fromFile: Path, lineNo: int, keyType: str, keyValue: str) -> None: + super().__init__( + fromFile, lineNo, + f'{keyType} "{keyValue}" is marked as DELETED and should not be used.' + ) class DeprecatedKeyError(BuildCacheBaseException): @@ -188,25 +184,24 @@ class DeprecatedKeyError(BuildCacheBaseException): name should not appear in the .csv file. """ - def __init__(self, fromFile, lineNo, keyType, keyValue, newValue): - super().__init__(fromFile, lineNo, - "{} '{}' is deprecated " - "and should be replaced with '{}'.".format( - keyType, keyValue, newValue - )) + def __init__(self, fromFile: Path, lineNo: int, keyType: str, keyValue: str, newValue: str) -> None: + super().__init__( + fromFile, lineNo, + f'{keyType} "{keyValue}" is deprecated and should be replaced with "{newValue}".' + ) class MultipleStationEntriesError(DuplicateKeyError): """ Raised when a station appears multiple times in the same file. """ - def __init__(self, fromFile, lineNo, facility, prevLineNo): + def __init__(self, fromFile: Path, lineNo: int, facility: str, prevLineNo: int) -> None: super().__init__(fromFile, lineNo, 'station', facility, prevLineNo) class MultipleItemEntriesError(DuplicateKeyError): """ Raised when one item appears multiple times in the same station. """ - def __init__(self, fromFile, lineNo, item, prevLineNo): + def __init__(self, fromFile: Path, lineNo: int, item: str, prevLineNo: int) -> None: super().__init__(fromFile, lineNo, 'item', item, prevLineNo) @@ -218,9 +213,8 @@ class SyntaxError(BuildCacheBaseException): text Offending text """ - def __init__(self, fromFile, lineNo, problem, text): - error = "{},\ngot: '{}'.".format(problem, text.strip()) - super().__init__(fromFile, lineNo, error) + def __init__(self, fromFile: Path, lineNo: int, problem: str, text: str) -> None: + super().__init__(fromFile, lineNo, f'{problem},\ngot: "{text.strip()}".') class SupplyError(BuildCacheBaseException): @@ -228,51 +222,80 @@ class SupplyError(BuildCacheBaseException): Raised when a supply field is incorrectly formatted. """ - def __init__(self, fromFile, lineNo, category, problem, value): - error = "Invalid {} supply value: {}. Got: {}". \ - format(category, problem, value) - super().__init__(fromFile, lineNo, error) + def __init__(self, fromFile: Path, lineNo: int, category: str, problem: str, value: Any) -> None: + super().__init__(fromFile, lineNo, f'Invalid {category} supply value: {problem}. Got: {value}') + ###################################################################### # Helpers -def parseSupply(pricesFile, lineNo, category, reading): +# supply/demand levels are one of '?' for unknown, 'L', 'M' or 'H' +# for low, medium, or high. We turn these into integer values for +# ordering convenience, and we include both upper and lower-case +# so we don't have to sweat ordering. +# +SUPPLY_LEVEL_VALUES = { + '?': -1, + 'L': 1, 'l': 1, + 'M': 2, 'm': 2, + 'H': 3, 'h': 3, +} + + +def parseSupply(pricesFile: Path, lineNo: int, category: str, reading: str) -> tuple[int, int]: + """ Parse a supply specifier which is expected to be in the , and + returns the units as an integer and a numeric level value suitable for ordering, + such that ? = -1, L/l = 0, M/m = 1, H/h = 2 """ + + # supply_level <- digit+ level; + # digit <- [0-9]; + # level <- Unknown / Low / Medium / High; + # Unknown <- '?'; + # Low <- 'L'; + # Medium <- 'M'; + # High <- 'H'; + if reading == '?': + return -1, -1 + elif reading == '-': + return 0, 0 + + # extract the left most digits into unit and the last character into the level reading. units, level = reading[0:-1], reading[-1] - levelNo = "??LMH".find(level.upper()) - 1 - if levelNo < -1: + + # Extract the right most character as the "level" and look up its numeric value. + levelNo = SUPPLY_LEVEL_VALUES.get(level) + if levelNo is None: raise SupplyError( pricesFile, lineNo, category, reading, - 'Unrecognized level suffix: "{}": ' - "expected one of 'L', 'M', 'H' or '?'".format( - level - ) + f'Unrecognized level suffix: "{level}": expected one of "L", "M", "H" or "?"' ) + + # Expecting a numeric value in units, e.g. 123? -> (units=123, level=?) try: unitsNo = int(units) if unitsNo < 0: - raise ValueError("unsigned unit count") - if unitsNo == 0: - return 0, 0 - return unitsNo, levelNo + # Use the same code-path as if the units fail to parse. + raise ValueError('negative unit count') except ValueError: - pass - - raise SupplyError( - pricesFile, lineNo, category, reading, - 'Unrecognized units/level value: "{}": ' - "expected '-', '?', or a number followed " - "by a level (L, M, H or ?).".format( - level - ) - ) + raise SupplyError( + pricesFile, lineNo, category, reading, + f'Unrecognized units/level value: "{level}": expected "-", "?", or a number followed by a level (L, M, H or ?).' + ) from None # don't forward the exception itself + + # Normalize the units and level when there are no units. + if unitsNo == 0: + return 0, 0 + + return unitsNo, levelNo + ###################################################################### # Code ###################################################################### -def getSystemByNameIndex(cur): +def getSystemByNameIndex(cur: sqlite3.Cursor) -> dict[str, int]: """ Build station index in STAR/Station notation """ cur.execute(""" SELECT system_id, UPPER(system.name) @@ -281,7 +304,7 @@ def getSystemByNameIndex(cur): return { name: ID for (ID, name) in cur } -def getStationByNameIndex(cur): +def getStationByNameIndex(cur: sqlite3.Cursor) -> dict[str, int]: """ Build station index in STAR/Station notation """ cur.execute(""" SELECT station_id, @@ -293,7 +316,7 @@ def getStationByNameIndex(cur): return { name.upper(): ID for (ID, name) in cur } -def getItemByNameIndex(cur): +def getItemByNameIndex(cur: sqlite3.Cursor) -> dict[str, int]: """ Generate item name index. """ @@ -301,10 +324,37 @@ def getItemByNameIndex(cur): return { name: itemID for (itemID, name) in cur } -def processPrices(tdenv, priceFile, db, defaultZero): +# The return type of process prices is complicated, should probably have been a type +# in its own right. I'm going to define some aliases to try and persuade IDEs to be +# more helpful about what it is trying to return. +if typing.TYPE_CHECKING: + # A list of the IDs of stations that were modified so they can be updated + ProcessedStationIds= tuple[tuple[int]] + ProcessedItem = tuple[ + int, # station ID + int, # item ID + Optional[int | float |str], # modified + int, # demandCR + int, # demandUnits + int, # demandLevel + int, # supplyCr + int, # supplyUnits + int, # supplyLevel + ] + ProcessedItems = list[ProcessedItem] + ZeroItems = list[tuple[int, int]] # stationID, itemID + + +def processPrices(tdenv: TradeEnv, priceFile: Path, db: sqlite3.Connection, defaultZero: bool) -> tuple[ProcessedStationIds, ProcessedItems, ZeroItems, int, int, int, int]: """ Yields SQL for populating the database with prices by reading the file handle for price lines. + + :param tdenv: The environment we're working in + :param priceFile: File to read + :param db: SQLite3 database to write to + :param defaultZero: Whether to create default zero-availability/-demand records for data that's not present + (if this is a partial update, you don't want this to be False) """ DEBUG0, DEBUG1 = tdenv.DEBUG0, tdenv.DEBUG1 @@ -340,20 +390,18 @@ def processPrices(tdenv, priceFile, db, defaultZero): processedSystems = set() processedItems = {} stationItemDates = {} - itemPrefix = "" DELETED = corrections.DELETED - items, zeros, buys, sells = [], [], [], [] + items, zeros = [], [] lineNo, localAdd = 0, 0 if not ignoreUnknown: - - def ignoreOrWarn(error): + def ignoreOrWarn(error: Exception) -> None: raise error elif not quiet: ignoreOrWarn = tdenv.WARN - def changeStation(matches): + def changeStation(matches: re.Match) -> None: nonlocal facility, stationID nonlocal processedStations, processedItems, localAdd nonlocal stationItemDates @@ -363,20 +411,21 @@ def changeStation(matches): systemNameIn, stationNameIn = matches.group(1, 2) systemName, stationName = systemNameIn.upper(), stationNameIn.upper() corrected = False - facility = "/".join((systemName, stationName)) + facility = f'{systemName}/{stationName}' # Make sure it's valid. stationID = DELETED - newID = stationByName.get(facility, -1) + newID = stationByName.get(facility, -1) # why -1 and not None? DEBUG0("Selected station: {}, ID={}", facility, newID) if newID is DELETED: DEBUG1("DELETED Station: {}", facility) return + if newID < 0: if utils.checkForOcrDerp(tdenv, systemName, stationName): return corrected = True - altName = sysCorrections.get(systemName, None) + altName = sysCorrections.get(systemName) if altName is DELETED: DEBUG1("DELETED System: {}", facility) return @@ -384,21 +433,22 @@ def changeStation(matches): DEBUG1("SYSTEM '{}' renamed '{}'", systemName, altName) systemName, facility = altName, "/".join((altName, stationName)) - systemID = systemByName.get(systemName, -1) + systemID = systemByName.get(systemName, -1) # why -1 and not None? if systemID < 0: ignoreOrWarn( UnknownSystemError(priceFile, lineNo, facility) ) return - altStation = stnCorrections.get(facility, None) - if altStation is DELETED: - DEBUG1("DELETED Station: {}", facility) - return + altStation = stnCorrections.get(facility) if altStation: + if altStation is DELETED: + DEBUG1("DELETED Station: {}", facility) + return + DEBUG1("Station '{}' renamed '{}'", facility, altStation) stationName = altStation.upper() - facility = "/".join((systemName, stationName)) + facility = f'{systemName}/{stationName}' newID = stationByName.get(facility, -1) if newID is DELETED: @@ -409,7 +459,7 @@ def changeStation(matches): if not ignoreUnknown: DEBUG0(f'Key value: "{list(stationByName.keys())[list(stationByName.values()).index(128893178)]}"') ignoreOrWarn( - UnknownStationError(priceFile, lineNo, facility) + UnknownStationError(priceFile, lineNo, facility) ) return name = utils.titleFixup(stationName) @@ -430,8 +480,8 @@ def changeStation(matches): """, [systemID, name]) newID = inscur.lastrowid stationByName[facility] = newID - tdenv.NOTE("Added local station placeholder for {} (#{})", - facility, newID + tdenv.NOTE( + "Added local station placeholder for {} (#{})", facility, newID ) localAdd += 1 elif newID in processedStations: @@ -452,7 +502,7 @@ def changeStation(matches): FROM StationItem WHERE station_id = ? """, [stationID]) - stationItemDates = {ID: modified for ID, modified in cur} + stationItemDates = dict(cur) addItem, addZero = items.append, zeros.append getItemID = itemByName.get @@ -496,7 +546,7 @@ def processItemLine(matches): ignoreOrWarn( MultipleItemEntriesError( priceFile, lineNo, - "{}".format(itemName), + f'{itemName}', processedItems[itemID] ) ) @@ -515,26 +565,16 @@ def processItemLine(matches): else: newItems += 1 if demandString: - if demandString == "?": - demandUnits, demandLevel = -1, -1 - elif demandString == "-": - demandUnits, demandLevel = 0, 0 - else: - demandUnits, demandLevel = parseSupply( - priceFile, lineNo, 'demand', demandString - ) + demandUnits, demandLevel = parseSupply( + priceFile, lineNo, 'demand', demandString + ) else: demandUnits, demandLevel = defaultUnits, defaultLevel if demandString and supplyString: - if supplyString == "?": - supplyUnits, supplyLevel = -1, -1 - elif supplyString == "-": - supplyUnits, supplyLevel = 0, 0 - else: - supplyUnits, supplyLevel = parseSupply( - priceFile, lineNo, 'supply', supplyString - ) + supplyUnits, supplyLevel = parseSupply( + priceFile, lineNo, 'supply', supplyString + ) else: supplyUnits, supplyLevel = defaultUnits, defaultLevel @@ -552,32 +592,24 @@ def processItemLine(matches): space_cleanup = re.compile(r'\s{2,}').sub for line in priceFile: lineNo += 1 - text, _, comment = line.partition('#') - text = text.strip() - # text = space_cleanup(text, ' ').strip() + + text = line.split('#', 1)[0] # Discard comments + text = space_cleanup(' ', text).strip() # Remove leading/trailing whitespace, reduce multi-spaces if not text: continue - # replace whitespace with single spaces - if text.find(" "): - # http://stackoverflow.com/questions/2077897 - text = ' '.join(text.split()) - ######################################## # ## "@ STAR/Station" lines. if text.startswith('@'): matches = systemStationRe.match(text) if not matches: - raise SyntaxError("Unrecognized '@' line: {}".format(# pylint: disable=no-value-for-parameter - text - )) + raise SyntaxError(priceFile, lineNo, "Unrecognized '@' line", text) changeStation(matches) continue if not stationID: # Need a station to process any other type of line. - raise SyntaxError(priceFile, lineNo, - "Expecting '@ SYSTEM / Station' line", text) + raise SyntaxError(priceFile, lineNo, "Expecting '@ SYSTEM / Station' line", text) if stationID == DELETED: # Ignore all values from a deleted station/system. continue @@ -592,8 +624,7 @@ def processItemLine(matches): # ## "Item sell buy ..." lines. matches = newItemPriceRe.match(text) if not matches: - raise SyntaxError(priceFile, lineNo, - "Unrecognized line/syntax", text) + raise SyntaxError(priceFile, lineNo, "Unrecognized line/syntax", text) processItemLine(matches) @@ -610,13 +641,14 @@ def processItemLine(matches): stations = tuple((ID,) for ID in processedStations.keys()) return stations, items, zeros, newItems, updtItems, ignItems, numSys + ###################################################################### -def processPricesFile(tdenv, db, pricesPath, pricesFh = None, defaultZero = False): +def processPricesFile(tdenv: TradeEnv, db: sqlite3.Connection, pricesPath: Path, pricesFh: Optional[TextIO] = None, defaultZero: bool = False) -> None: tdenv.DEBUG0("Processing Prices file '{}'", pricesPath) - with pricesFh or pricesPath.open('r', encoding = 'utf-8') as pricesFh: + with pricesFh or pricesPath.open('r', encoding='utf-8') as pricesFh: stations, items, zeros, newItems, updtItems, ignItems, numSys = processPrices( tdenv, pricesFh, db, defaultZero ) @@ -663,7 +695,6 @@ def processPricesFile(tdenv, db, pricesPath, pricesFh = None, defaultZero = Fals # ?, ?, ? # ) # """, items) - updatedItems = len(items) tdenv.DEBUG0("Marking populated stations as having a market") db.execute( @@ -674,8 +705,9 @@ def processPricesFile(tdenv, db, pricesPath, pricesFh = None, defaultZero = Fals ")" ) - tdenv.DEBUG0(f'Committing...') + tdenv.DEBUG0('Committing...') db.commit() + db.close() changes = " and ".join("{} {}".format(v, k) for k, v in { "new": newItems, @@ -696,6 +728,7 @@ def processPricesFile(tdenv, db, pricesPath, pricesFh = None, defaultZero = Fals if ignItems: tdenv.NOTE("Ignored {} items with old data", ignItems) + ###################################################################### @@ -749,26 +782,27 @@ def processImportFile(tdenv, db, importPath, tableName): str(importPath), tableName ) - fkeySelectStr = ("(" - "SELECT {newValue}" - " FROM {table}" - " WHERE {stmt}" - ")" + fkeySelectStr = ( + "(" + " SELECT {newValue}" + " FROM {table}" + " WHERE {stmt}" + ")" ) uniquePfx = "unq:" uniqueLen = len(uniquePfx) ignorePfx = "!" - with importPath.open('r', encoding = 'utf-8') as importFile: + with importPath.open('r', encoding='utf-8') as importFile: csvin = csv.reader( - importFile, delimiter = ',', quotechar = "'", doublequote = True + importFile, delimiter=',', quotechar="'", doublequote=True ) # first line must be the column names columnDefs = next(csvin) columnCount = len(columnDefs) # split up columns and values - # this is necessqary because the insert might use a foreign key + # this is necessary because the insert might use a foreign key bindColumns = [] bindValues = [] joinHelper = [] @@ -817,10 +851,10 @@ def processImportFile(tdenv, db, importPath, tableName): sql_stmt = """ INSERT OR REPLACE INTO {table} ({columns}) VALUES({values}) """.format( - table = tableName, - columns = ','.join(bindColumns), - values = ','.join(bindValues) - ) + table=tableName, + columns=','.join(bindColumns), + values=','.join(bindValues) + ) tdenv.DEBUG0("SQL-Statement: {}", sql_stmt) # Check if there is a deprecation check for this table. @@ -832,7 +866,7 @@ def processImportFile(tdenv, db, importPath, tableName): # import the data importCount = 0 - uniqueIndex = dict() + uniqueIndex = {} for linein in csvin: if not linein: @@ -968,6 +1002,7 @@ def buildCache(tdb, tdenv): tempDB.commit() tempDB.close() + tdb.close() tdenv.DEBUG0("Swapping out db files") @@ -1021,7 +1056,7 @@ def importDataFromFile(tdb, tdenv, path, pricesFh = None, reset = False): db = tdb.getDB(), pricesPath = path, pricesFh = pricesFh, - ) + ) # If everything worked, we may need to re-build the prices file. if path != tdb.pricesPath: diff --git a/tradedangerous/cli.py b/tradedangerous/cli.py index 329ef471..d5ae37e0 100644 --- a/tradedangerous/cli.py +++ b/tradedangerous/cli.py @@ -31,11 +31,6 @@ # cool, please see the TradeDB and TradeCalc modules. TD is designed # to empower other programmers to do cool stuff. -from __future__ import absolute_import -from __future__ import with_statement -from __future__ import print_function -from __future__ import division -from __future__ import unicode_literals import os import traceback @@ -58,7 +53,7 @@ def main(argv = None): "\tBug Tracker: https://github.com/eyeonus/Trade-Dangerous/issues\n" "\tDocumentation: https://github.com/eyeonus/Trade-Dangerous/wiki\n" "\tEDForum Thread: https://forums.frontier.co.uk/showthread.php/441509\n" - ) + ) from . import tradeexcept try: @@ -78,7 +73,7 @@ def main(argv = None): if 'EXCEPTIONS' in os.environ: raise e sys.exit(1) - except (UnicodeEncodeError, UnicodeDecodeError) as e: + except (UnicodeEncodeError, UnicodeDecodeError): print("-----------------------------------------------------------") print("ERROR: Unexpected unicode error in the wild!") print() diff --git a/tradedangerous/commands/TEMPLATE.py b/tradedangerous/commands/TEMPLATE.py index 5aa49af9..45b7b98e 100644 --- a/tradedangerous/commands/TEMPLATE.py +++ b/tradedangerous/commands/TEMPLATE.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals from commands.commandenv import ResultRow from commands.parsing import * from formatting import RowFormat, ColumnFormat @@ -59,4 +58,4 @@ def render(results, cmdenv, tdb): This is where you should generate any output from your command. """ - ### TODO: Implement \ No newline at end of file + ### TODO: Implement diff --git a/tradedangerous/commands/__init__.py b/tradedangerous/commands/__init__.py index 9f37c9ad..446cd179 100644 --- a/tradedangerous/commands/__init__.py +++ b/tradedangerous/commands/__init__.py @@ -1,9 +1,7 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals from .commandenv import CommandEnv from textwrap import TextWrapper import argparse # For parsing command line args. -import importlib import os import pathlib import sys @@ -63,7 +61,7 @@ def addArguments(group, options, required, topGroup = None): parsing.registerParserHelpers(exGrp) addArguments(exGrp, option.arguments, required, topGroup = group) else: - assert not required in option.kwargs + assert required not in option.kwargs if option.args[0][0] == '-': group.add_argument(*(option.args), required = required, **(option.kwargs)) else: @@ -84,7 +82,7 @@ def _findFromFile(cmd, prefix = '.tdrc'): return None -class CommandIndex(object): +class CommandIndex: def usage(self, argv): """ diff --git a/tradedangerous/commands/buildcache_cmd.py b/tradedangerous/commands/buildcache_cmd.py index 100fb980..5b188b66 100644 --- a/tradedangerous/commands/buildcache_cmd.py +++ b/tradedangerous/commands/buildcache_cmd.py @@ -1,8 +1,6 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals -import sys - +from __future__ import annotations from .exceptions import CommandLineError -from .parsing import * +from .parsing import ParseArgument from ..cache import buildCache from ..tradedb import TradeDB @@ -54,19 +52,16 @@ # Perform query and populate result set -def run(results, cmdenv, tdb): +def run(results, cmdenv, tdb: TradeDB): # Check that the file doesn't already exist. if not cmdenv.force: if tdb.dbPath.exists(): raise CommandLineError( - "SQLite3 database '{}' already exists.\n" - "Either remove the file first or use the '-f' option." - .format(tdb.dbFilename)) + "SQLite3 database '{tdb.dbFilename}' already exists.\n" + "Either remove the file first or use the '-f' option.") if not tdb.sqlPath.exists(): - raise CommandLineError( - "SQL File does not exist: {}" - .format(tdb.sqlFilename)) + raise CommandLineError("SQL File does not exist: {tdb.sqlFilename}") buildCache(tdb, cmdenv) diff --git a/tradedangerous/commands/buy_cmd.py b/tradedangerous/commands/buy_cmd.py index 5c7bcc77..a8c8ec6f 100644 --- a/tradedangerous/commands/buy_cmd.py +++ b/tradedangerous/commands/buy_cmd.py @@ -1,12 +1,14 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals +from __future__ import annotations from collections import defaultdict from .commandenv import ResultRow -from .exceptions import * -from .parsing import * -from ..formatting import RowFormat, ColumnFormat, max_len -from ..tradedb import TradeDB, AmbiguityError, System, Station +from .exceptions import CommandLineError, NoDataError +from ..formatting import RowFormat, max_len +from ..tradedb import Station, System, TradeDB +from .parsing import ( + AvoidPlacesArgument, BlackMarketSwitch, FleetCarrierArgument, MutuallyExclusiveGroup, + NoPlanetSwitch, OdysseyArgument, PadSizeArgument, ParseArgument, PlanetaryArgument, +) -import math ITEM_MODE = "Item" SHIP_MODE = "Ship" @@ -114,7 +116,7 @@ def get_lookup_list(cmdenv, tdb): name for names in cmdenv.name for name in names.split(',') ] # We only support searching for one type of purchase a time: ship or item. - # Our first match is open ended, but once we have matched one type of + # Our first match is open-ended, but once we have matched one type of # thing, the remaining arguments are all sourced from the same pool. # Thus: [food, cobra, metals] is illegal but [metals, hydrogen] is legal. mode = None @@ -401,8 +403,5 @@ def render(results, cmdenv, tdb): print(stnRowFmt.format(row)) if singleMode and cmdenv.detail: - print("{:{lnl}} {:>10n}".format( - "-- Ship Cost" if mode is SHIP_MODE else "-- Average", - results.summary.avg, - lnl = maxStnLen, - )) + msg = "-- Ship Cost" if mode is SHIP_MODE else "-- Average", + print(f"{msg:{maxStnLen}} {results.summary.avg:>10n}") diff --git a/tradedangerous/commands/commandenv.py b/tradedangerous/commands/commandenv.py index 24417049..ffcbbf31 100644 --- a/tradedangerous/commands/commandenv.py +++ b/tradedangerous/commands/commandenv.py @@ -1,7 +1,8 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals - -from .exceptions import CommandLineError, PadSizeError, PlanetaryError, FleetCarrierError -from ..tradedb import AmbiguityError, System, Station +from .exceptions import ( + CommandLineError, FleetCarrierError, OdysseyError, + PadSizeError, PlanetaryError, +) +from ..tradedb import AmbiguityError, Station from ..tradeenv import TradeEnv import os @@ -9,7 +10,7 @@ import sys -class CommandResults(object): +class CommandResults: """ Encapsulates the results returned by running a command. """ @@ -24,7 +25,7 @@ def render(self, cmdenv = None, tdb = None): cmdenv._cmd.render(self, cmdenv, tdb) -class ResultRow(object): +class ResultRow: def __init__(self, **kwargs): for k, v in kwargs.items(): @@ -46,7 +47,7 @@ def __init__(self, properties, argv, cmdModule): if self.detail and self.quiet: raise CommandLineError("'--detail' (-v) and '--quiet' (-q) are mutually exclusive.") - self._cmd = cmdModule or __main__ + self._cmd = cmdModule or getattr("__main__") self.wantsTradeDB = getattr(cmdModule, 'wantsTradeDB', True) self.usesTradeData = getattr(cmdModule, 'usesTradeData', False) @@ -187,7 +188,7 @@ def checkAvoids(self): # But if it matched more than once, whine about ambiguity if item and place: - raise AmbiguityError('Avoidance', avoid, [ item, place.str() ]) + raise AmbiguityError('Avoidance', avoid, [ item, place.text() ]) self.DEBUG0("Avoiding items {}, places {}", [ item.name() for item in avoidItems ], @@ -213,7 +214,7 @@ def checkPadSize(self): return self.padSize = padSize = padSize.upper() for value in padSize: - if not value in 'SML?': + if value not in 'SML?': raise PadSizeError(padSize) self.padSize = padSize @@ -227,7 +228,7 @@ def checkPlanetary(self): return self.planetary = planetary = planetary.upper() for value in planetary: - if not value in 'YN?': + if value not in 'YN?': raise PlanetaryError(planetary) self.planetary = planetary @@ -237,8 +238,8 @@ def checkFleet(self): return fleet = ''.join(sorted(list(set(fleet)))).upper() for value in fleet: - if not value in 'YN?': - raise FleetError(fleet) + if value not in 'YN?': + raise FleetCarrierError(fleet) if fleet == '?NY': self.fleet = None return @@ -250,12 +251,12 @@ def checkOdyssey(self): return odyssey = ''.join(sorted(list(set(odyssey)))).upper() for value in odyssey: - if not value in 'YN?': - raise odysseyError(odyssey) + if value not in 'YN?': + raise OdysseyError(odyssey) if odyssey == '?NY': self.odyssey = None return - self.odyssey = odyssey = odyssey.upper() + self.odyssey = odyssey.upper() def colorize(self, color, rawText): """ diff --git a/tradedangerous/commands/exceptions.py b/tradedangerous/commands/exceptions.py index fe25a989..efa204ac 100644 --- a/tradedangerous/commands/exceptions.py +++ b/tradedangerous/commands/exceptions.py @@ -1,6 +1,5 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals from ..tradeexcept import TradeException -import sys + ###################################################################### # Exceptions @@ -8,6 +7,7 @@ class UsageError(TradeException): def __init__(self, title, usage): self.title, self.usage = title, usage + def __str__(self): return self.title + "\n\n" + self.usage @@ -20,6 +20,7 @@ class CommandLineError(TradeException): """ def __init__(self, errorStr, usage=None): self.errorStr, self.usage = errorStr, usage + def __str__(self): if self.usage: return "ERROR: {}\n\n{}".format(self.errorStr, self.usage) @@ -35,8 +36,9 @@ class NoDataError(TradeException): """ def __init__(self, errorStr): self.errorStr = errorStr + def __str__(self): - return "Error: {}\n".format(self.errorStr) + (""" + return f"""Error: {self.errorStr} Possible causes: - No profitable runs or routes meet your criteria, - Missing Systems or Stations along the route (see "local -vv"), @@ -47,7 +49,7 @@ def __str__(self): For more help, see the TradeDangerous Wiki: https://github.com/eyeonus/Trade-Dangerous/wiki -""").format(sys.argv[0]) +""" class PadSizeError(CommandLineError): diff --git a/tradedangerous/commands/export_cmd.py b/tradedangerous/commands/export_cmd.py index f7a7aecb..c0cdc481 100644 --- a/tradedangerous/commands/export_cmd.py +++ b/tradedangerous/commands/export_cmd.py @@ -1,7 +1,5 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals - from ..csvexport import exportTableToFile -from .parsing import * +from .parsing import ParseArgument, MutuallyExclusiveGroup from .exceptions import CommandLineError from pathlib import Path @@ -122,4 +120,4 @@ def run(results, cmdenv, tdb): filePath.unlink() cmdenv.DEBUG0("Delete empty file {file}'".format(file=filePath)) - return None \ No newline at end of file + return None diff --git a/tradedangerous/commands/import_cmd.py b/tradedangerous/commands/import_cmd.py index 722c297f..bdc3fd8c 100644 --- a/tradedangerous/commands/import_cmd.py +++ b/tradedangerous/commands/import_cmd.py @@ -1,13 +1,11 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals - -from .exceptions import * -from .parsing import * +from .exceptions import CommandLineError +from .parsing import ParseArgument, MutuallyExclusiveGroup from itertools import chain from pathlib import Path from .. import cache, plugins, transfers -import math import re +import sys try: import tkinter diff --git a/tradedangerous/commands/local_cmd.py b/tradedangerous/commands/local_cmd.py index 74fa41f5..9712ae07 100644 --- a/tradedangerous/commands/local_cmd.py +++ b/tradedangerous/commands/local_cmd.py @@ -1,12 +1,14 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals from .commandenv import ResultRow -from .parsing import * +from .parsing import ( + ParseArgument, PadSizeArgument, MutuallyExclusiveGroup, NoPlanetSwitch, + PlanetaryArgument, FleetCarrierArgument, OdysseyArgument, BlackMarketSwitch, + ShipyardSwitch, OutfittingSwitch, RearmSwitch, RefuelSwitch, RepairSwitch, +) from ..formatting import RowFormat, ColumnFormat, max_len from itertools import chain from ..tradedb import TradeDB from ..tradeexcept import TradeException -import math ###################################################################### # Parser config @@ -186,48 +188,37 @@ def render(results, cmdenv, tdb): key=lambda row: row.age) ).append( ColumnFormat("Mkt", '>', '3', - key=lambda row: \ - TradeDB.marketStates[row.station.market]) + key=lambda row: TradeDB.marketStates[row.station.market]) ).append( ColumnFormat("BMk", '>', '3', - key=lambda row: \ - TradeDB.marketStates[row.station.blackMarket]) + key=lambda row: TradeDB.marketStates[row.station.blackMarket]) ).append( ColumnFormat("Shp", '>', '3', - key=lambda row: \ - TradeDB.marketStates[row.station.shipyard]) + key=lambda row: TradeDB.marketStates[row.station.shipyard]) ).append( ColumnFormat("Out", '>', '3', - key=lambda row: \ - TradeDB.marketStates[row.station.outfitting]) + key=lambda row: TradeDB.marketStates[row.station.outfitting]) ).append( ColumnFormat("Arm", '>', '3', - key=lambda row: \ - TradeDB.marketStates[row.station.rearm]) + key=lambda row: TradeDB.marketStates[row.station.rearm]) ).append( ColumnFormat("Ref", '>', '3', - key=lambda row: \ - TradeDB.marketStates[row.station.refuel]) + key=lambda row: TradeDB.marketStates[row.station.refuel]) ).append( ColumnFormat("Rep", '>', '3', - key=lambda row: \ - TradeDB.marketStates[row.station.repair]) + key=lambda row: TradeDB.marketStates[row.station.repair]) ).append( ColumnFormat("Pad", '>', '3', - key=lambda row: \ - TradeDB.padSizes[row.station.maxPadSize]) + key=lambda row: TradeDB.padSizes[row.station.maxPadSize]) ).append( ColumnFormat("Plt", '>', '3', - key=lambda row: \ - TradeDB.planetStates[row.station.planetary]) + key=lambda row: TradeDB.planetStates[row.station.planetary]) ).append( ColumnFormat("Flc", '>', '3', - key=lambda row: \ - TradeDB.fleetStates[row.station.fleet]) + key=lambda row: TradeDB.fleetStates[row.station.fleet]) ).append( ColumnFormat("Ody", '>', '3', - key=lambda row: \ - TradeDB.odysseyStates[row.station.odyssey]) + key=lambda row: TradeDB.odysseyStates[row.station.odyssey]) ) if cmdenv.detail > 1: stnRowFmt.append( diff --git a/tradedangerous/commands/market_cmd.py b/tradedangerous/commands/market_cmd.py index 7144fcee..256c061f 100644 --- a/tradedangerous/commands/market_cmd.py +++ b/tradedangerous/commands/market_cmd.py @@ -1,9 +1,10 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals from .commandenv import ResultRow -from .exceptions import * -from .parsing import * -from ..formatting import RowFormat, ColumnFormat -from ..tradedb import TradeDB +from .exceptions import CommandLineError +from .parsing import ( + ParseArgument, MutuallyExclusiveGroup, +) +from ..formatting import RowFormat + ###################################################################### # Parser config @@ -129,8 +130,8 @@ def render(results, cmdenv, tdb): if showCategories: rowFmt.prefix = ' ' - sellPred = lambda row: row.sellCr != 0 and row.supply != '-' - buyPred = lambda row: row.buyCr != 0 and row.demand != '-' + sellPred = lambda row: row.sellCr != 0 and row.supply != '-' # noqa: E731 + buyPred = lambda row: row.buyCr != 0 and row.demand != '-' # noqa: E731 rowFmt.addColumn('Item', '<', longestLen, key=lambda row: row.item.name()) @@ -170,4 +171,4 @@ def render(results, cmdenv, tdb): if showCategories and row.item.category is not lastCat: print("+{}".format(row.item.category.name())) lastCat = row.item.category - print(rowFmt.format(row)) \ No newline at end of file + print(rowFmt.format(row)) diff --git a/tradedangerous/commands/nav_cmd.py b/tradedangerous/commands/nav_cmd.py index 943afc98..421fde61 100644 --- a/tradedangerous/commands/nav_cmd.py +++ b/tradedangerous/commands/nav_cmd.py @@ -1,9 +1,12 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals -from .parsing import * -import math +from .parsing import ( + AvoidPlacesArgument, FleetCarrierArgument, MutuallyExclusiveGroup, + NoPlanetSwitch, OdysseyArgument, PadSizeArgument, ParseArgument, + PlanetaryArgument, +) from ..tradedb import System, Station, TradeDB from ..tradeexcept import TradeException + ###################################################################### # Parser config @@ -197,48 +200,37 @@ def render(results, cmdenv, tdb): key=lambda row: row.age) ).append( ColumnFormat('Mkt', '>', '3', - key=lambda row: \ - TradeDB.marketStates[row.station.market]) + key=lambda row: TradeDB.marketStates[row.station.market]) ).append( ColumnFormat("BMk", '>', '3', - key=lambda row: \ - TradeDB.marketStates[row.station.blackMarket]) + key=lambda row: TradeDB.marketStates[row.station.blackMarket]) ).append( ColumnFormat("Shp", '>', '3', - key=lambda row: \ - TradeDB.marketStates[row.station.shipyard]) + key=lambda row: TradeDB.marketStates[row.station.shipyard]) ).append( ColumnFormat("Out", '>', '3', - key=lambda row: \ - TradeDB.marketStates[row.station.outfitting]) + key=lambda row: TradeDB.marketStates[row.station.outfitting]) ).append( ColumnFormat("Arm", '>', '3', - key=lambda row: \ - TradeDB.marketStates[row.station.rearm]) + key=lambda row: TradeDB.marketStates[row.station.rearm]) ).append( ColumnFormat("Ref", '>', '3', - key=lambda row: \ - TradeDB.marketStates[row.station.refuel]) + key=lambda row: TradeDB.marketStates[row.station.refuel]) ).append( ColumnFormat("Rep", '>', '3', - key=lambda row: \ - TradeDB.marketStates[row.station.repair]) + key=lambda row: TradeDB.marketStates[row.station.repair]) ).append( ColumnFormat("Pad", '>', '3', - key=lambda row: \ - TradeDB.padSizes[row.station.maxPadSize]) + key=lambda row: TradeDB.padSizes[row.station.maxPadSize]) ).append( ColumnFormat("Plt", '>', '3', - key=lambda row: \ - TradeDB.planetStates[row.station.planetary]) + key=lambda row: TradeDB.planetStates[row.station.planetary]) ).append( ColumnFormat("Flc", '>', '3', - key=lambda row: \ - TradeDB.fleetStates[row.station.fleet]) + key=lambda row: TradeDB.fleetStates[row.station.fleet]) ).append( ColumnFormat("Ody", '>', '3', - key=lambda row: \ - TradeDB.odysseyStates[row.station.odyssey]) + key=lambda row: TradeDB.odysseyStates[row.station.odyssey]) ) if cmdenv.detail > 1: stnRowFmt.append( diff --git a/tradedangerous/commands/olddata_cmd.py b/tradedangerous/commands/olddata_cmd.py index 26292a2c..e552c5c2 100644 --- a/tradedangerous/commands/olddata_cmd.py +++ b/tradedangerous/commands/olddata_cmd.py @@ -1,11 +1,10 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals -from .parsing import * +from .parsing import ( + FleetCarrierArgument, MutuallyExclusiveGroup, NoPlanetSwitch, + OdysseyArgument, ParseArgument, PadSizeArgument, PlanetaryArgument, +) from ..tradedb import TradeDB from ..tradeexcept import TradeException -import itertools -import math -import sys ###################################################################### # Parser config @@ -66,7 +65,6 @@ def run(results, cmdenv, tdb): cmdenv = results.cmdenv tdb = cmdenv.tdb - srcSystem = cmdenv.nearSystem results.summary = ResultRow() results.limit = cmdenv.limit @@ -240,20 +238,16 @@ def render(results, cmdenv, tdb): key=lambda row: row.station.distFromStar()) ).append( ColumnFormat("Pad", '>', '3', - key=lambda row: \ - TradeDB.padSizes[row.station.maxPadSize]) + key=lambda row: TradeDB.padSizes[row.station.maxPadSize]) ).append( ColumnFormat("Plt", '>', '3', - key=lambda row: \ - TradeDB.planetStates[row.station.planetary]) + key=lambda row: TradeDB.planetStates[row.station.planetary]) ).append( ColumnFormat("Flc", '>', '3', - key=lambda row: \ - TradeDB.fleetStates[row.station.fleet]) + key=lambda row: TradeDB.fleetStates[row.station.fleet]) ).append( ColumnFormat("Ody", '>', '3', - key=lambda row: \ - TradeDB.odysseyStates[row.station.odyssey]) + key=lambda row: TradeDB.odysseyStates[row.station.odyssey]) ) if not cmdenv.quiet: @@ -261,4 +255,4 @@ def render(results, cmdenv, tdb): print(heading, underline, sep='\n') for row in results.rows: - print(rowFmt.format(row)) \ No newline at end of file + print(rowFmt.format(row)) diff --git a/tradedangerous/commands/parsing.py b/tradedangerous/commands/parsing.py index bca55167..31178b1a 100644 --- a/tradedangerous/commands/parsing.py +++ b/tradedangerous/commands/parsing.py @@ -1,10 +1,11 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals -from .exceptions import PadSizeError, PlanetaryError, FleetCarrierError +from .exceptions import ( + FleetCarrierError, OdysseyError, PadSizeError, PlanetaryError, +) ###################################################################### # Parsing Helpers -class ParseArgument(object): +class ParseArgument: """ Provides argument forwarding so that 'makeSubParser' can take function-like arguments. """ @@ -12,7 +13,7 @@ def __init__(self, *args, **kwargs): self.args, self.kwargs = args, kwargs -class MutuallyExclusiveGroup(object): +class MutuallyExclusiveGroup: def __init__(self, *args): self.arguments = list(args) @@ -24,9 +25,10 @@ def __init__(self, *args): class CreditParser(int): """ argparse helper for parsing numeric prefixes, i.e. - 'k' for thousand, 'm' for million and 'b' for billion. + 'k' for thousands, 'm' for millions and 'b' for billions. """ suffixes = {'k': 10**3, 'm': 10**6, 'b': 10**9} + def __new__(cls, val, **kwargs): if isinstance(val, str): if val[-1] in CreditParser.suffixes: @@ -82,7 +84,7 @@ def __init__(self, help=None): else: self.args = (self.switches,) help = help or self.help - self.kwargs = {'action':'store_true', 'dest':self.dest, 'help':help} + self.kwargs = {'action': 'store_true', 'dest': self.dest, 'help': help} class BlackMarketSwitch(SwitchArgument): @@ -209,6 +211,7 @@ def __init__(self): 'choices': 'YN?', } + __tdParserHelpers = { 'credits': CreditParser, 'padsize': PadSizeArgument.PadSizeParser, diff --git a/tradedangerous/commands/rares_cmd.py b/tradedangerous/commands/rares_cmd.py index e80cc986..2a9172e0 100644 --- a/tradedangerous/commands/rares_cmd.py +++ b/tradedangerous/commands/rares_cmd.py @@ -1,11 +1,12 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals from .commandenv import ResultRow -from .exceptions import * -from .parsing import * -from ..formatting import RowFormat, ColumnFormat, max_len +from .exceptions import CommandLineError +from .parsing import ( + PadSizeArgument, ParseArgument, MutuallyExclusiveGroup, NoPlanetSwitch, + PlanetaryArgument, FleetCarrierArgument, OdysseyArgument, +) +from ..formatting import RowFormat, max_len from ..tradedb import TradeDB -import math ###################################################################### # Parser config @@ -61,8 +62,7 @@ default=False, ), ParseArgument('--away', - help='Require "--from" systems to be at least this far ' \ - 'from primary system', + help='Require "--from" systems to be at least this far from primary system', metavar='LY', default=0, type=float, @@ -78,8 +78,7 @@ ) ), ParseArgument('--from', - help='Additional systems to range check candidates against, ' \ - 'requires --away.', + help='Additional systems to range check candidates against, requires --away.', metavar='SYSTEMNAME', action='append', dest='awayFrom', @@ -165,7 +164,7 @@ def run(results, cmdenv, tdb): # Look through the rares list. for rare in tdb.rareItemByID.values(): - if not rare.illegal in wantIllegality: + if rare.illegal not in wantIllegality: continue if padSize: # do we care about pad size? if not rare.station.checkPadSize(padSize): diff --git a/tradedangerous/commands/run_cmd.py b/tradedangerous/commands/run_cmd.py index baab2d28..3ced88ad 100644 --- a/tradedangerous/commands/run_cmd.py +++ b/tradedangerous/commands/run_cmd.py @@ -1,12 +1,17 @@ from .commandenv import ResultRow -from .exceptions import * -from .parsing import * +from .exceptions import CommandLineError, NoDataError +from .parsing import ( + BlackMarketSwitch, FleetCarrierArgument, MutuallyExclusiveGroup, + NoPlanetSwitch, OdysseyArgument, PadSizeArgument, ParseArgument, + PlanetaryArgument, +) from itertools import chain -from ..formatting import RowFormat, ColumnFormat from ..tradedb import TradeDB, System, Station, describeAge from ..tradecalc import TradeCalc, Route, NoHopsError import math +import sys + ###################################################################### # Parser config @@ -220,8 +225,7 @@ dest = 'x52pro', ), ParseArgument('--prune-score', - help = 'From the 3rd hop on, only consider routes which have ' \ - 'at least this percentage of the current best route''s score.', + help = 'From the 3rd hop on, only consider routes which have at least this percentage of the current best route''s score.', dest = 'pruneScores', type = float, default = 0, @@ -261,7 +265,7 @@ # Helpers -class Checklist(object): +class Checklist: """ Class for encapsulating display of a route as a series of steps to be 'checked off' as the user passes through them. @@ -294,13 +298,13 @@ def note(self, str, addBreak = True): print("(i) {} (i){}".format(str, "\n" if addBreak else "")) def run(self, route, cr): - tdb, mfd = self.tdb, self.mfd + mfd = self.mfd stations, hops, jumps = route.route, route.hops, route.jumps lastHopIdx = len(stations) - 1 gainCr = 0 self.stepNo = 0 - heading = "(i) BEGINNING CHECKLIST FOR {} (i)".format(route.str(lambda x, y : y)) + heading = "(i) BEGINNING CHECKLIST FOR {} (i)".format(route.text(lambda x, y: y)) print(heading, "\n", '-' * len(heading), "\n\n", sep = '') cmdenv = self.cmdenv @@ -313,8 +317,8 @@ def run(self, route, cr): cur, nxt, hop = stations[idx], stations[idx + 1], hops[idx] sortedTradeOptions = sorted( hop[0], - key = lambda tradeOption: \ - tradeOption[1] * tradeOption[0].gainCr, reverse = True + key=lambda tradeOption: tradeOption[1] * tradeOption[0].gainCr, + reverse=True ) # Tell them what they need to buy. @@ -345,7 +349,7 @@ def run(self, route, cr): for jump in jumps[idx][1:]: self.doStep('Jump to', jump.name()) if cmdenv.detail: - self.doStep('Dock at', nxt.str()) + self.doStep('Dock at', nxt.text()) print() self.note("Sell at {}".format(nxt.name())) @@ -413,9 +417,9 @@ def expandForJumps(tdb, cmdenv, calc, origin, jumps, srcName, purpose): [sys.dbname for sys in origins] ) thisJump, origins = origins, set() - for sys in thisJump: - avoid.add(sys) - for stn in sys.stations or (): + for system in thisJump: + avoid.add(system) + for stn in system.stations or (): if stn.ID not in tradingList: cmdenv.DEBUG2( "X {}/{} not in trading list", @@ -433,7 +437,7 @@ def expandForJumps(tdb, cmdenv, calc, origin, jumps, srcName, purpose): stn.system.dbname, stn.dbname, ) stations.add(stn) - for dest, dist in tdb.genSystemsInRange(sys, maxLyPer): + for dest, dist in tdb.genSystemsInRange(system, maxLyPer): if dest not in avoid: origins.add(dest) @@ -463,7 +467,7 @@ def expandForJumps(tdb, cmdenv, calc, origin, jumps, srcName, purpose): ) stations = list(stations) - stations.sort(key = lambda stn: stn.ID) + stations.sort(key=lambda stn: stn.ID) return stations @@ -666,16 +670,12 @@ def filterStationSet(src, cmdenv, calc, stnList): src, ",".join(station.name() for station in stnList), ) - filtered = tuple( + stnList = tuple( place for place in stnList - if isinstance(place, System) or \ - checkStationSuitability(cmdenv, calc, place, src) + if isinstance(place, System) or checkStationSuitability(cmdenv, calc, place, src) ) if not stnList: - raise CommandLineError( - "No {} station met your criteria.".format( - src - )) + raise CommandLineError("No {src} station met your criteria.") return stnList @@ -969,9 +969,9 @@ def validateRunArguments(tdb, cmdenv, calc): raise CommandLineError("Can't have same from/to with --unique") if viaSet: if len(origins) == 1 and origins[0] in viaSet: - raise("Can't have --from station in --via list with --unique") + raise CommandLineError("Can't have --from station in --via list with --unique") if len(destns) == 1 and destns[0] in viaSet: - raise("Can't have --to station in --via list with --unique") + raise CommandLineError("Can't have --to station in --via list with --unique") if cmdenv.mfd: cmdenv.mfd.display("Loading Trades") @@ -1147,7 +1147,6 @@ def run(results, cmdenv, tdb): validateRunArguments(tdb, cmdenv, calc) origPlace, viaSet = cmdenv.origPlace, cmdenv.viaSet - avoidPlaces = cmdenv.avoidPlaces stopStations = cmdenv.destinations goalSystem = cmdenv.goalSystem maxLs = cmdenv.maxLs diff --git a/tradedangerous/commands/sell_cmd.py b/tradedangerous/commands/sell_cmd.py index a767dde2..9851ca0e 100644 --- a/tradedangerous/commands/sell_cmd.py +++ b/tradedangerous/commands/sell_cmd.py @@ -1,9 +1,13 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals -from .exceptions import * -from .parsing import * +from .commandenv import ResultRow +from .exceptions import CommandLineError, NoDataError +from .parsing import ( + AvoidPlacesArgument, BlackMarketSwitch, FleetCarrierArgument, + MutuallyExclusiveGroup, NoPlanetSwitch, OdysseyArgument, + PadSizeArgument, ParseArgument, PlanetaryArgument, +) from ..tradedb import TradeDB, System, Station +from ..formatting import RowFormat -import math ###################################################################### # Parser config @@ -70,9 +74,7 @@ ###################################################################### # Perform query and populate result set -def run(results, cmdenv, tdb:TradeDB): - from .commandenv import ResultRow - +def run(results, cmdenv, tdb: TradeDB): if cmdenv.lt and cmdenv.gt: if cmdenv.lt <= cmdenv.gt: raise CommandLineError("--gt must be lower than --lt") @@ -199,8 +201,6 @@ def run(results, cmdenv, tdb:TradeDB): ## Transform result set into output def render(results, cmdenv, tdb): - from ..formatting import RowFormat, ColumnFormat - longestNamed = max(results.rows, key=lambda result: len(result.station.name())) longestNameLen = len(longestNamed.station.name()) diff --git a/tradedangerous/commands/shipvendor_cmd.py b/tradedangerous/commands/shipvendor_cmd.py index 0038fd24..e0d8674e 100644 --- a/tradedangerous/commands/shipvendor_cmd.py +++ b/tradedangerous/commands/shipvendor_cmd.py @@ -1,18 +1,13 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals from .commandenv import ResultRow from .exceptions import CommandLineError -from .parsing import * +from .parsing import MutuallyExclusiveGroup, ParseArgument from ..formatting import RowFormat, ColumnFormat, max_len from itertools import chain -from ..tradedb import AmbiguityError -from ..tradedb import System, Station -from ..tradedb import TradeDB +from ..tradedb import Station # Original by Dirk Wilhelm from .. import csvexport -import re -import sys ###################################################################### # Parser config @@ -211,6 +206,8 @@ def run(results, cmdenv, tdb): for ship in ships.values(): if action(tdb, cmdenv, station, ship): dataToExport = True + + cmdenv.DEBUG0("dataToExport = {}", dataToExport) maybeExportToCSV(tdb, cmdenv) diff --git a/tradedangerous/commands/station_cmd.py b/tradedangerous/commands/station_cmd.py index 8aa0dc4e..b3624380 100644 --- a/tradedangerous/commands/station_cmd.py +++ b/tradedangerous/commands/station_cmd.py @@ -1,9 +1,8 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals from .commandenv import ResultRow from .exceptions import CommandLineError -from .parsing import * +from .parsing import MutuallyExclusiveGroup, ParseArgument from ..tradedb import AmbiguityError -from ..tradedb import System, Station +from ..tradedb import Station from ..tradedb import TradeDB from .. import utils from ..formatting import max_len @@ -12,7 +11,7 @@ from .. import csvexport import difflib import re -import sys + ###################################################################### # Parser config @@ -244,9 +243,7 @@ def checkSystemAndStation(tdb, cmdenv): sysName = None if not stnName: - raise CommandLineError("Invalid station name: {}".format( - envStnName - )) + raise CommandLineError("Invalid station name: {stnName}") if not sysName: raise CommandLineError("No system name specified") @@ -346,7 +343,6 @@ def run(results, cmdenv, tdb): system, station = checkSystemAndStation(tdb, cmdenv) - systemName = cmdenv.system stationName = cmdenv.station if cmdenv.add: @@ -367,7 +363,7 @@ def run(results, cmdenv, tdb): avgSell = results.summary.avgSelling = tdb.getAverageSelling() avgBuy = results.summary.avgBuying = tdb.getAverageBuying() - class ItemTrade(object): + class ItemTrade: def __init__(self, ID, price, avgAgainst): self.ID, self.item = ID, tdb.itemByID[ID] self.price = int(price) @@ -492,11 +488,9 @@ def makeBest(rows, explanation, alt, maxLen, starFn): ) print("Best Buy..:", makeBest( results.summary.selling, "Buy from this station", "Sell", longestNameLen, - starFn=lambda price, avgCr: \ - price <= (avgCr * 0.9), + starFn=lambda price, avgCr: price <= (avgCr * 0.9), )) print("Best Sale.:", makeBest( results.summary.buying, "Sell to this station", "Cost", longestNameLen, - starFn=lambda price, avgCr: \ - price >= (avgCr * 1.1), - )) \ No newline at end of file + starFn=lambda price, avgCr: price >= (avgCr * 1.1), + )) diff --git a/tradedangerous/commands/trade_cmd.py b/tradedangerous/commands/trade_cmd.py index 3eef6ba0..2dbde446 100644 --- a/tradedangerous/commands/trade_cmd.py +++ b/tradedangerous/commands/trade_cmd.py @@ -1,10 +1,7 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals -from .exceptions import * -from .parsing import * -from ..tradedb import TradeDB -from ..tradecalc import TradeCalc, Route - -import math +from .exceptions import CommandLineError +from .parsing import ParseArgument +from ..tradecalc import TradeCalc +from ..formatting import RowFormat, max_len ###################################################################### # Parser config @@ -65,8 +62,6 @@ def run(results, cmdenv, tdb): ## Transform result set into output def render(results, cmdenv, tdb): - from ..formatting import RowFormat, ColumnFormat, max_len - longestNameLen = max_len(results.rows, key=lambda row: row.item.name(cmdenv.detail)) rowFmt = RowFormat() @@ -102,4 +97,4 @@ def render(results, cmdenv, tdb): print(heading, underline, sep='\n') for row in results.rows: - print(rowFmt.format(row)) \ No newline at end of file + print(rowFmt.format(row)) diff --git a/tradedangerous/commands/update_cmd.py b/tradedangerous/commands/update_cmd.py index 14d6a5df..33498866 100644 --- a/tradedangerous/commands/update_cmd.py +++ b/tradedangerous/commands/update_cmd.py @@ -1,8 +1,7 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals -from .parsing import * +from .parsing import MutuallyExclusiveGroup, ParseArgument from ..tradeexcept import TradeException from .exceptions import CommandLineError -from ..tradedb import System, Station +from ..tradedb import System from .. import prices, cache import subprocess import os @@ -167,7 +166,8 @@ def getEditorPaths(cmdenv, editorName, envVar, windowsFolders, winExe, nixExe): cmdenv.DEBUG0("Locating {} editor", editorName) try: return os.environ[envVar] - except KeyError: pass + except KeyError: + pass paths = [] @@ -183,7 +183,8 @@ def getEditorPaths(cmdenv, editorName, envVar, windowsFolders, winExe, nixExe): try: paths += os.environ['PATH'].split(os.pathsep) - except KeyError: pass + except KeyError: + pass for path in paths: candidate = os.path.join(path, binary) @@ -271,8 +272,10 @@ def editUpdate(tdb, cmdenv, stationID): dbFilename = tdb.dbFilename try: elementMask = prices.Element.basic | prices.Element.supply - if cmdenv.timestamps: elementMask |= prices.Element.timestamp - if cmdenv.all: elementMask |= prices.Element.blanks + if cmdenv.timestamps: + elementMask |= prices.Element.timestamp + if cmdenv.all: + elementMask |= prices.Element.blanks # Open the file and dump data to it. with tmpPath.open("w", encoding = 'utf-8') as tmpFile: # Remember the filename so we know we need to delete it. diff --git a/tradedangerous/commands/update_gui.py b/tradedangerous/commands/update_gui.py index 6d838fa2..0d37f36d 100644 --- a/tradedangerous/commands/update_gui.py +++ b/tradedangerous/commands/update_gui.py @@ -1,9 +1,7 @@ import tkinter as tk import tkinter.messagebox as mbox -import tkinter.ttk as ttk import sqlite3 import re -from pathlib import Path """ This is a crude attempt at a GUI for updating trade prices. @@ -44,7 +42,7 @@ "- Use Tab, Shift-Tab, Up/Down Arrow and Enter to navigate.\n" ) -class Item(object): +class Item: """ Describe a listed, tradeable item """ def __init__(self, ID, catID, name, displayNo): diff --git a/tradedangerous/corrections.py b/tradedangerous/corrections.py index bb6455c4..226261a1 100644 --- a/tradedangerous/corrections.py +++ b/tradedangerous/corrections.py @@ -1,8 +1,6 @@ # Provides an interface for correcting names that # have changed in recent versions. -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals - # Arbitrary, negative value to denote something that's been removed. DELETED = -111 @@ -26,7 +24,7 @@ 'COOLING HOSES': 'Micro-weave Cooling Hoses', 'METHANOL MONOHYDRATE': 'Methanol Monohydrate Crystals', 'OCCUPIED CRYOPOD': 'Occupied Escape Pod', - 'SALVAGEABLE WRECKAGE': 'Wreckage Components', + 'SALVAGEABLE WRECKAGE': 'Wreckage Components', 'POLITICAL PRISONER': 'Political Prisoners', 'HOSTAGE': 'Hostages', "VOID OPAL": "Void Opals", diff --git a/tradedangerous/csvexport.py b/tradedangerous/csvexport.py index 01fd0d77..57294e4e 100644 --- a/tradedangerous/csvexport.py +++ b/tradedangerous/csvexport.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals - from pathlib import Path from .tradeexcept import TradeException @@ -34,9 +32,10 @@ # Helpers ###################################################################### -def search_keyList(list, val): - for row in list: - if row['from'] == row['to'] == val: return row +def search_keyList(items, val): + for row in items: + if row['from'] == row['to'] == val: + return row def getUniqueIndex(conn, tableName): """ return all unique columns """ @@ -134,13 +133,14 @@ def exportTableToFile(tdb, tdenv, tableName, csvPath=None): pkCount = 0 for columnRow in cur.execute("PRAGMA table_info('%s')" % tableName): # count the columns of the primary key - if columnRow['pk'] > 0: pkCount += 1 + if columnRow['pk'] > 0: + pkCount += 1 # build column list columnList = [] for columnRow in cur.execute("PRAGMA table_info('%s')" % tableName): # if there is only one PK column, ignore it - #if columnRow['pk'] > 0 and pkCount == 1: continue + # if columnRow['pk'] > 0 and pkCount == 1: continue columnList.append(columnRow) if len(columnList) == 0: @@ -199,7 +199,7 @@ def exportTableToFile(tdb, tdenv, tableName, csvPath=None): stmtColumn += [ "{}.{}".format(tableName, col['name']) ] # build the SQL statement - sqlStmt = "SELECT {} FROM {}".format(",".join(stmtColumn)," ".join(stmtTable)) + sqlStmt = "SELECT {} FROM {}".format(",".join(stmtColumn), " ".join(stmtTable)) if len(stmtOrder) > 0: sqlStmt += " ORDER BY {}".format(",".join(stmtOrder)) tdenv.DEBUG1("SQL: %s" % sqlStmt) diff --git a/tradedangerous/edscupdate.py b/tradedangerous/edscupdate.py index cbe48f0e..ed29e320 100644 --- a/tradedangerous/edscupdate.py +++ b/tradedangerous/edscupdate.py @@ -25,7 +25,6 @@ """ import argparse -import math import misc.clipboard import misc.edsc import os @@ -158,7 +157,7 @@ def parse_arguments(): default=2, ) grp = parser.add_mutually_exclusive_group() - if grp: # for indentation + if grp: # for indentation grp.add_argument( '--random', action='store_true', @@ -297,7 +296,7 @@ def check_database(tdb, name, x, y, z): WHERE pos_x BETWEEN ? and ? AND pos_y BETWEEN ? and ? AND pos_z BETWEEN ? and ? - """, [ + """, [ x - 0.5, x + 0.5, y - 0.5, y + 0.5, z - 0.5, z + 0.5, @@ -391,9 +390,7 @@ def main(): edsq.status['statusnum'], )) - date = data['date'] systems = data['systems'] - print("{} results".format(len(systems))) # Filter out systems we already know that match the EDSC data. systems = [ @@ -534,7 +531,7 @@ def main(): continue if ok.startswith('='): name = ok[1:].strip().upper() - if not name in extras: + if name not in extras: add_to_extras(argv, name) ok = 'y' if ok.lower() != 'y': @@ -555,6 +552,7 @@ def main(): submit_distance(argv, name, distance) + if __name__ == "__main__": try: main() diff --git a/tradedangerous/edsmupdate.py b/tradedangerous/edsmupdate.py index f76f1a03..87574e2d 100644 --- a/tradedangerous/edsmupdate.py +++ b/tradedangerous/edsmupdate.py @@ -27,7 +27,6 @@ """ import argparse -import math import misc.clipboard import misc.edsm import os @@ -67,7 +66,7 @@ def parse_arguments(): default=os.environ.get('CMDR', None), ) grp = parser.add_mutually_exclusive_group() - if grp: # for indentation + if grp: # for indentation grp.add_argument( '--random', action='store_true', @@ -271,7 +270,7 @@ def main(): ) if not argv.date: - argv.date = tdb.query("SELECT MAX(modified) FROM System").fetchone()[0] + argv.date = tdb.query("SELECT MAX(modified) FROM System").fetchone()[0] dateRe = re.compile(r'^20\d\d-(0[1-9]|1[012])-(0[1-9]|[12]\d|3[01]) ([01]\d|2[0123]):[0-5]\d:[0-5]\d$') if not dateRe.match(argv.date): raise UsageError( @@ -440,7 +439,7 @@ def main(): break if ok.startswith('='): name = ok[1:].strip().upper() - if not name in extras: + if name not in extras: add_to_extras(argv, name) ok = 'y' if ok.lower() != 'y': @@ -462,6 +461,7 @@ def main(): if argv.add and not commit: tdb.getDB().commit() + if __name__ == "__main__": try: main() diff --git a/tradedangerous/formatting.py b/tradedangerous/formatting.py index 4e13de29..1dc907c0 100644 --- a/tradedangerous/formatting.py +++ b/tradedangerous/formatting.py @@ -1,13 +1,22 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals +""" +Provides a library of mechanisms for formatting output text to ensure consistency across +TradeDangerous and plugin tools, +""" +from __future__ import annotations + import itertools +import typing + +if typing.TYPE_CHECKING: + from typing import Any, Callable, Optional -class ColumnFormat(object): +class ColumnFormat: """ Describes formatting of a column to be populated with data. Member Functions: - str() + text() Applies all formatting (except qualifier) to the name to produce a correctly sized title field. @@ -47,7 +56,7 @@ class ColumnFormat(object): ] rows = [ {'name':'Bob', 'dist':1.5}, {'name':'John', 'dist':23}] # print titles - print(*[col.str() for col in cols]) + print(*[col.text() for col in cols]) for row in rows: print(*[col.format(row) for col in cols]) Produces: @@ -56,6 +65,15 @@ class ColumnFormat(object): John [23.00] """ + name: str # name of the column + align: str # format's alignment specifier + width: int # width specifier + qualifier: Optional[str] # optional format type specifier e.g. '.2f', 's', 'n' + pre: Optional[str] # prefix to the column + post: Optional[str] # postfix to the column + key: Callable # function to retrieve the printable name of the item + pred: Callable # predicate: return False to leave this column blank + def __init__( self, name, @@ -66,7 +84,7 @@ def __init__( post=None, key=lambda item: item, pred=lambda item: True, - ): + ) -> None: self.name = name self.align = align self.width = max(int(width), len(name)) @@ -76,29 +94,17 @@ def __init__( self.post = post or '' self.pred = pred - def str(self): - return '{pre}{title:{align}{width}}{post}'.format( - title=self.name, - align=self.align, width=self.width, - pre=self.pre, post=self.post, - ) - - def format(self, value): - if self.pred(value): - return '{pre}{value:{align}{width}{qual}}{post}'.format( - value=self.key(value), - align=self.align, width=self.width, - qual=self.qualifier, - pre=self.pre, post=self.post, - ) - else: - return '{pre}{value:{align}{width}}{post}'.format( - value="", - align=self.align, width=self.width, - pre=self.pre, post=self.post, - ) + def __str__(self) -> str: + return f'{self.pre}{self.name:{self.align}{self.width}}{self.post}' + text = __str__ + + def format(self, value: str) -> str: + """ Returns the string formatted with a specific value""" + if not self.pred(value): + return f'{self.pre}{"":{self.align}{self.width}}{self.post}' + return f'{self.pre}{self.key(value):{self.align}{self.width}{self.qualifier}}{self.post}' -class RowFormat(object): +class RowFormat: """ Describes an ordered collection of ColumnFormats for dispay data from rows, such that calling @@ -117,7 +123,7 @@ class RowFormat(object): insert(pos, newCol) Inserts a ColumnFormatter at position pos in the list - str() + text() Returns a list of all the column headings format(rowData): @@ -125,14 +131,17 @@ class RowFormat(object): of the columns """ - def __init__(self, prefix=None): + columns: list[ColumnFormat] + prefix: str + + def __init__(self, prefix: Optional[str] = None): self.columns = [] self.prefix = prefix or "" - def addColumn(self, *args, **kwargs): + def addColumn(self, *args, **kwargs) -> None: self.append(ColumnFormat(*args, **kwargs)) - def append(self, column, after=None): + def append(self, column: ColumnFormat, after: Optional[str] = None) -> 'RowFormat': columns = self.columns if after: for idx, col in enumerate(columns, 1): @@ -142,19 +151,22 @@ def append(self, column, after=None): columns.append(column) return self - def insert(self, pos, column): + def insert(self, pos: int, column: Optional[ColumnFormat]) -> None: if column is not None: self.columns.insert(pos, column) - def str(self): - return self.prefix + ' '.join(col.str() for col in self.columns) + def __str__(self) -> str: + return f"{self.prefix} {' '.join(str(col) for col in self.columns)}" + + text = __str__ # alias - def heading(self): - headline = self.str() + def heading(self) -> tuple[str, str]: + """ Returns a title and the appropriate underline for that text. """ + headline = f"{self}" return headline, '-' * len(headline) - def format(self, rowData): - return self.prefix + ' '.join(col.format(rowData) for col in self.columns) + def format(self, row_data: Optional[Any]) -> str: + return f"{self.prefix} {' '.join(col.format(row_data) for col in self.columns)}" def max_len(iterable, key=lambda item: item): iterable, readahead = itertools.tee(iter(iterable)) @@ -164,6 +176,7 @@ def max_len(iterable, key=lambda item: item): return 0 return max(len(key(item)) for item in iterable) + if __name__ == '__main__': rowFmt = RowFormat(). \ append(ColumnFormat("Name", '<', '8', key=lambda row: row['name'])). \ @@ -175,7 +188,7 @@ def max_len(iterable, key=lambda item: item): ] def present(): - rowTitle = rowFmt.str() + rowTitle = rowFmt.text() print(rowTitle) print('-' * len(rowTitle)) for row in rows: @@ -188,4 +201,4 @@ def present(): print("Adding age ColumnFormat:") rowFmt.append(after='Name', col=ColumnFormat("Age", '>', 3, pre='|', post='|', key=lambda row: row['age'])) - present() \ No newline at end of file + present() diff --git a/tradedangerous/fs.py b/tradedangerous/fs.py index 94341fc0..a0cdda8e 100644 --- a/tradedangerous/fs.py +++ b/tradedangerous/fs.py @@ -1,7 +1,7 @@ """This module should handle filesystem related operations """ from shutil import copy as shcopy -from os import makedirs, path, utime +from os import makedirs from pathlib import Path __all__ = ['copy', 'copyallfiles', 'touch', 'ensurefolder'] @@ -25,7 +25,7 @@ def copy(src, dst): def copy_if_newer(src, dst): """ - copy src to dst if src is newer + copy src to dst if src is newer takes string or Path object as input returns Path(dst) on success returns Path(src) if not newer @@ -33,11 +33,11 @@ def copy_if_newer(src, dst): """ srcPath = pathify(src).resolve() dstPath = pathify(dst) - if dstPath.exists() and not (dstPath.stat().st_mtime < srcPath.stat().st_mtime): + if dstPath.exists() and dstPath.stat().st_mtime >= srcPath.stat().st_mtime: return srcPath - else: - shcopy(str(srcPath), str(dstPath)) - return dstPath + + shcopy(str(srcPath), str(dstPath)) + return dstPath def copyallfiles(srcdir, dstdir): """ diff --git a/tradedangerous/gui.py b/tradedangerous/gui.py index bed72d3d..4e09ea3c 100644 --- a/tradedangerous/gui.py +++ b/tradedangerous/gui.py @@ -31,12 +31,6 @@ # individual always-on-top for every window # Data retrieval from CMDR's journal -from __future__ import absolute_import -from __future__ import with_statement -from __future__ import print_function -from __future__ import division -from __future__ import unicode_literals -from pkg_resources import resource_filename import os import sys import traceback @@ -47,7 +41,6 @@ from appJar import gui import appJar - # from tkinter import * # import tkinter.font as font # import tkinter.scrolledtext as scrolledtext @@ -60,8 +53,6 @@ from . import tradedb from .plugins import PluginException -from pycparser.ply.yacc import MAXINT -from pkg_resources import _sset_none # ================== # BEGIN appJar fixes @@ -96,8 +87,9 @@ def setSpinBoxPos(self, title, pos, callFunction = True): vals = self._getSpinBoxValsAsList(vals) pos = int(pos) if pos < 0 or pos >= len(vals): - raise Exception("Invalid position: " + str(pos) + ". No position in SpinBox: " + - title + "=" + str(vals)) + raise RuntimeError( + f"Invalid position: {pos}. No position in SpinBox: {title}={vals}" + ) # pos = len(vals) - 1 - pos val = vals[pos] self._setSpinBoxVal(spin, val, callFunction) @@ -140,10 +132,10 @@ def _configOptionBoxList(self, title, options, kind): # get the longest string length try: maxSize = len(str(max(options, key = len))) - except: + except: # noqa: E722 try: maxSize = len(str(max(options))) - except: + except: # noqa: E722 maxSize = 0 # increase if ticks @@ -220,9 +212,9 @@ def changeCWD(): """ Opens a folder select dialog for choosing the current working directory. """ - cwd = filedialog.askdirectory(title = "Select the top-level folder for TD to work in...", + cwd = filedialog.askdirectory(title = "Select the top-level folder for TD to work in...", initialdir = argVals['--cwd']) - #cwd = win.directoryBox("Select the top-level folder for TD to work in...", dirName = argVals['--cwd']) + # cwd = win.directoryBox("Select the top-level folder for TD to work in...", dirName = argVals['--cwd']) if cwd: argVals['--cwd'] = str(Path(cwd)) widgets['cwd']['text'] = argVals['--cwd'] @@ -231,13 +223,14 @@ def changeDB(): """ Opens a file select dialog for choosing the database file. """ - db = filedialog.askopenfilename(title = "Select the TD database file to use...", + db = filedialog.askopenfilename(title = "Select the TD database file to use...", initialdir = str(Path(argVals['--db']).parent), filetypes = [('Data Base File', '*.db')]) if db: argVals['--db'] = str(Path(db)) widgets['db']['text'] = argVals['--db'] + # A dict of all arguments in TD (mostly auto-generated) # Manually add the global arguments for now, maybe figure out how to auto-populate them as well. allArgs = { @@ -300,16 +293,16 @@ def buildArgDicts(): # print(arg.args[0]) argVals[arg.args[0]] = arg.kwargs.get('default') or None - allArgs[cmd]['req'][arg.args[0]] = {kwarg : arg.kwargs[kwarg] for kwarg in arg.kwargs} + allArgs[cmd]['req'][arg.args[0]] = {kwarg: arg.kwargs[kwarg] for kwarg in arg.kwargs} allArgs[cmd]['req'][arg.args[0]]['widget'] = chooseType(arg) # print(allArgs[cmd]['req']) if index.switches: for arg in index.switches: try: - argVals[arg.args[0]] = value = arg.kwargs.get('default') or None + argVals[arg.args[0]] = arg.kwargs.get('default') or None - allArgs[cmd]['opt'][arg.args[0]] = {kwarg : arg.kwargs[kwarg] for kwarg in arg.kwargs} + allArgs[cmd]['opt'][arg.args[0]] = {kwarg: arg.kwargs[kwarg] for kwarg in arg.kwargs} allArgs[cmd]['opt'][arg.args[0]]['widget'] = chooseType(arg) if arg.args[0] == '--option': @@ -324,51 +317,49 @@ def buildArgDicts(): except AttributeError: for argGrp in arg.arguments: - argVals[argGrp.args[0]] = value = argGrp.kwargs.get('default') or None + argVals[argGrp.args[0]] = argGrp.kwargs.get('default') or None - allArgs[cmd]['opt'][argGrp.args[0]] = {kwarg : argGrp.kwargs[kwarg] for kwarg in argGrp.kwargs} + allArgs[cmd]['opt'][argGrp.args[0]] = {kwarg: argGrp.kwargs[kwarg] for kwarg in argGrp.kwargs} allArgs[cmd]['opt'][argGrp.args[0]]['widget'] = chooseType(argGrp) allArgs[cmd]['opt'][argGrp.args[0]]['excludes'] = [excl.args[0] for excl in arg.arguments if excl.args[0] != argGrp.args[0]] if argGrp.args[0] == '--plug': # Currently only the 'import' cmd has the '--plug' option, - # but this could no longer be the case in future. + # but this could no longer be the case in the future. if cmd == 'import': allArgs[cmd]['opt'][argGrp.args[0]]['plugins'] = importPlugs - # print(allArgs[cmd]['opt']) - # print(allArgs) - # print(argVals) + def optWindow(): """ Opens a window listing all of the options for the currently selected plugin. """ # with win.subWindow("Plugin Options", modal = True) as sw: - # win.emptyCurrentContainer() - # optDict = {} - # if argVals['--option']: - # for option in enumerate(argVals['--option'].split(',')): - # if '=' in option[1]: - # optDict[option[1].split('=')[0]] = option[1].split('=')[1] - # else: - # if option[1] != '': - # optDict[option[1]] = True - # # print(optDict) - # if not win.combo('--plug'): - # win.message('No import plugin chosen.', width = 170, colspan = 10) - # else: - # plugOpts = allArgs['import']['opt']['--option']['options'][win.combo('--plug')] - # for option in plugOpts: - # # print(option + ': ' + plugOpts[option]) - # if '=' in plugOpts[option]: - # win.entry(option, optDict.get(option) or '', label = True, sticky = 'ew', colspan = 10, tooltip = plugOpts[option]) - # else: - # win.check(option, optDict.get(option) or False, sticky = 'ew', colspan = 10, tooltip = plugOpts[option]) - # # print(plugOpts) - # win.button("Done", setOpts, column = 8) - # win.button("Cancel", sw.hide, row = 'p', column = 9) - # sw.show() + # win.emptyCurrentContainer() + # optDict = {} + # if argVals['--option']: + # for option in enumerate(argVals['--option'].split(',')): + # if '=' in option[1]: + # optDict[option[1].split('=')[0]] = option[1].split('=')[1] + # else: + # if option[1] != '': + # optDict[option[1]] = True + # # print(optDict) + # if not win.combo('--plug'): + # win.message('No import plugin chosen.', width = 170, colspan = 10) + # else: + # plugOpts = allArgs['import']['opt']['--option']['options'][win.combo('--plug')] + # for option in plugOpts: + # # print(option + ': ' + plugOpts[option]) + # if '=' in plugOpts[option]: + # win.entry(option, optDict.get(option) or '', label = True, sticky = 'ew', colspan = 10, tooltip = plugOpts[option]) + # else: + # win.check(option, optDict.get(option) or False, sticky = 'ew', colspan = 10, tooltip = plugOpts[option]) + # # print(plugOpts) + # win.button("Done", setOpts, column = 8) + # win.button("Cancel", sw.hide, row = 'p', column = 9) + # sw.show() def chooseType(arg): """ @@ -637,7 +628,7 @@ def updateCommandBox(args = None): # Setup the CLI interface and build the main window def main(argv = None): - class IORedirector(object): + class IORedirector: def __init__(self, TEXT_INFO): self.TEXT_INFO = TEXT_INFO @@ -672,7 +663,7 @@ def setOpts(): argStr = argStr.rsplit(',', 1)[0] win.entry('--option', argStr) sw.hide() - + #TODO: Implement in tk def optionsWin(): """ @@ -703,7 +694,7 @@ def optionsWin(): win.button("Done", setOpts, column = 8) win.button("Cancel", sw.hide, row = 'p', column = 9) sw.show() - + #TODO: Implement in tk def updArgs(name): """ @@ -749,7 +740,7 @@ def getWidgetType(name): win.setEntry(exclude, '', callFunction = False) elif widgetType == 'check': win.check(exclude, False, callFunction = False) - + #TODO: REMOVE def updCmd(): """ @@ -774,7 +765,7 @@ def updCmd(): win.label('Optional:', sticky = 'w') for key in allArgs[cmd]['opt']: makeWidgets(key, allArgs[cmd]['opt'][key]) - + def runTD(): """ Executes the TD command selected in the GUI. @@ -881,7 +872,7 @@ def runTrade(): win.message('outputText', '') threading.Thread(target = runTrade, name = "TDThread", daemon = True).start() - + # TODO: replace def makeWidgets(name, arg, sticky = 'ew', label = True, **kwargs): kwargs['sticky'] = sticky @@ -961,7 +952,7 @@ def makeWidgets(name, arg, sticky = 'ew', label = True, **kwargs): buildArgDicts() - + # window = Tk() # window.title('Trade Dangerous GUI (Beta), TD v.%s' % (__version__,)) # window.iconbitmap(resource_filename(__name__, "../tradedangerouscrest.ico")) @@ -1007,23 +998,23 @@ def makeWidgets(name, arg, sticky = 'ew', label = True, **kwargs): stretch = 'none', sticky = 'ew', width = 10, row = 0, column = 0, colspan = 5) with win.scrollPane('req', disabled = 'horizontal', row = 1, column = 0, colspan = 10) as pane: pane.configure(width = 200, height = 75) - + with win.scrollPane('opt', disabled = 'horizontal', row = 2, column = 0, colspan = 10) as pane: pane.configure(width = 200, height = 345) - + with win.tabbedFrame('tabFrame', disabled = 'horizontal', row = 1, column = 10, rowspan = 2, colspan = 40) as tabFrame: with win.tab('Help'): with win.scrollPane('helpPane', disabled = 'horizontal') as pane: pane.configure(width = 560, height = 420) win.message('helpText', cmdHelp['help']) win.widgetManager.get(WIDGET_NAMES.Message, 'helpText').config(width = 560) - + with win.tab('Output'): with win.scrollPane('outPane', disabled = 'horizontal') as pane: pane.configure(width = 560, height = 420) win.message('outputText', '') win.widgetManager.get(WIDGET_NAMES.Message, 'outputText').config(width = 560) - + makeWidgets('--link-ly', allArgs['--link-ly'], sticky = 'w', width = 4, row = 3, column = 2) makeWidgets('--quiet', allArgs['--quiet'], sticky = 'e', disabled = ':', width = 1, row = 3, column = 46) @@ -1034,12 +1025,12 @@ def makeWidgets(name, arg, sticky = 'ew', label = True, **kwargs): win.button('Run', runTD, tooltip = 'Execute the selected command.', sticky = 'w', row = 3, column = 49) - + makeWidgets('--cwd', allArgs['--cwd'], width = 4, row = 4, column = 0) with win.scrollPane('CWD', disabled = 'vertical', row = 4, column = 1, colspan = 49) as pane: pane.configure(width = 500, height = 20) widgets['cwd'] = win.label('cwd', argVals['--cwd'], sticky = 'w') - + makeWidgets('--db', allArgs['--db'], width = 4, row = 5, column = 0) with win.scrollPane('DB', disabled = 'vertical', row = 5, column = 1, colspan = 49) as pane: pane.configure(width = 500, height = 20) diff --git a/tradedangerous/jsonprices.py b/tradedangerous/jsonprices.py index 5b3a1b7c..aab0796a 100644 --- a/tradedangerous/jsonprices.py +++ b/tradedangerous/jsonprices.py @@ -24,7 +24,7 @@ def lookup_system(tdb, tdenv, name, x, y, z): pass if system: - if (system.posX != x or system.posY != y or system.posZ != z): + if system.posX != x or system.posY != y or system.posZ != z: raise Exception("System {} position mismatch: " "Got {},{},{} expected {},{},{}".format( name, @@ -33,16 +33,9 @@ def lookup_system(tdb, tdenv, name, x, y, z): )) return system - newSystem = "@{} [{}, {}, {}]".format( - name, x, y, z - ) - candidates = [] for candidate in tdb.systemByID.values(): - if (candidate.posX == x and - candidate.posY == y and - candidate.posZ == z - ): + if candidate.posX == x and candidate.posY == y and candidate.posZ == z: candidates.append(candidate) if len(candidates) == 1: @@ -61,10 +54,9 @@ def lookup_system(tdb, tdenv, name, x, y, z): )) return candidates[0] - if len(candidates): - raise Exception("System {} matches co-ordinates for systems: {}" + - ','.join([system.name for system in candidates]) - ) + if candidates: + options = ', '.join([s.name for s in candidates]) + raise RuntimeError(f"System {system.name} matches co-ordinates for systems: {options}") if tdenv.addUnknown: return tdb.addLocalSystem(name, x, y, z) @@ -91,7 +83,7 @@ def lookup_station( # Now set the parameters tdb.updateLocalStation( - stn, lsFromStar, blackMarket, maxPadSize + station, lsFromStar, blackMarket, maxPadSize ) return station @@ -116,7 +108,7 @@ def load_prices_json( try: blackMarket = stnData['bm'].upper() - if not blackMarket in [ 'Y', 'N' ]: + if blackMarket not in [ 'Y', 'N' ]: blackMarket = '?' except KeyError: blackMarket = '?' @@ -137,7 +129,7 @@ def load_prices_json( return if system.dbname != sysName and tdenv.detail: print("NOTE: Treating '{}' as '{}'".format( - name, system.dbname + sysName, system.dbname )) tdenv.DEBUG1("- System: {}", system.dbname) diff --git a/tradedangerous/mapping.py b/tradedangerous/mapping.py index b78fc144..7224ad54 100644 --- a/tradedangerous/mapping.py +++ b/tradedangerous/mapping.py @@ -2,7 +2,7 @@ # Mapping class for FDEV-IDs to TD names # -class FDEVMappingBase(object): +class FDEVMappingBase: """ Base class to map FDEV-IDs to TD names, do not use directly. @@ -67,7 +67,8 @@ def mapLoad(self): else: entries[ID] = {} for i, val in enumerate(line[1:], start=1): - if val: entries[ID][self.colNames[i]] = val + if val: + entries[ID][self.colNames[i]] = val self.tdenv.DEBUG2("{}: {}".format(ID, str(entries[ID]).replace("{", "{{").replace("}", "}}"))) self.entries = entries self.tdenv.DEBUG1("Loaded {:n} {}-Mappings".format(len(entries), self.tableName)) @@ -126,6 +127,6 @@ class FDEVMappingOutfitting(FDEVMappingBase): Maps ID to EDDN outfitting """ tableName = "FDevOutfitting" - colNames = [ 'id', 'category' , 'name', 'mount', + colNames = [ 'id', 'category', 'name', 'mount', 'guidance', 'ship', 'class', 'rating' ] diff --git a/tradedangerous/mfd/__init__.py b/tradedangerous/mfd/__init__.py index 36775d94..f4aa3480 100644 --- a/tradedangerous/mfd/__init__.py +++ b/tradedangerous/mfd/__init__.py @@ -2,13 +2,11 @@ # You are free to use, redistribute, or even print and eat a copy of # this software so long as you include this copyright notice. # I guarantee there is at least one bug neither of us knew about. -#--------------------------------------------------------------------- +# --------------------------------------------------------------------- # TradeDangerous :: Modules :: Multi-function display wrapper # # Multi-Function Display wrappers -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals - ###################################################################### # imports @@ -28,7 +26,7 @@ class MissingDeviceError(Exception): ###################################################################### # classes -class DummyMFD(object): +class DummyMFD: """ Base class for the MFD drivers, implemented as no-ops so that you can always use all MFD functions without conditionals. diff --git a/tradedangerous/mfd/saitek/__init__.py b/tradedangerous/mfd/saitek/__init__.py index c9fb219d..da65ff0b 100644 --- a/tradedangerous/mfd/saitek/__init__.py +++ b/tradedangerous/mfd/saitek/__init__.py @@ -1,4 +1,3 @@ # Saitek MFD wrappers __all__ = [ "DirectOutput", "X52Pro" ] - diff --git a/tradedangerous/mfd/saitek/directoutput.py b/tradedangerous/mfd/saitek/directoutput.py index 83db4a74..5de82475 100644 --- a/tradedangerous/mfd/saitek/directoutput.py +++ b/tradedangerous/mfd/saitek/directoutput.py @@ -60,8 +60,6 @@ """ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals - from tradedangerous.mfd import MissingDeviceError import ctypes @@ -69,6 +67,8 @@ import logging import os import platform +import sys +import time S_OK = 0x00000000 E_HANDLE = 0x80070006 @@ -84,8 +84,7 @@ SOFTBUTTON_DOWN = 0x00000004 -class DirectOutput(object): - +class DirectOutput: def __init__(self, dll_path): """ Creates python object to interact with DirecOutput.dll @@ -165,7 +164,7 @@ def RegisterSoftButtonCallback(self, device_handle, function): S_OK: The call completed successfully. E_HANDLE: The device handle specified is invalid. """ - + logging.debug("DirectOutput.RegisterSoftButtonCallback({}, {})".format(device_handle, function)) return self.DirectOutputDLL.DirectOutput_RegisterSoftButtonCallback(ctypes.wintypes.HANDLE(device_handle), function, 0) @@ -276,9 +275,9 @@ def SetString(self, device_handle, page, line, string): return self.DirectOutputDLL.DirectOutput_SetString(ctypes.wintypes.HANDLE(device_handle), page, line, len(string), ctypes.wintypes.LPWSTR(string)) -class DirectOutputDevice(object): +class DirectOutputDevice: - class Buttons(object): + class Buttons: select, up, down = False, False, False @@ -658,8 +657,6 @@ def __str__(self): # logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)s [%(filename)s:%(lineno)d] %(message)s') logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(name)s [%(filename)s:%(lineno)d] %(message)s') - import time, sys - device = DirectOutputDevice(debug_level=1) print("Device initialized") @@ -675,7 +672,7 @@ def __str__(self): while True: try: time.sleep(1) - except: - #This is used to catch Ctrl+C, calling finish method is *very* important to de-initalize device. + except: # noqa: E722 + # This is used to catch Ctrl+C, calling finish method is *very* important to de-initalize device. device.finish() sys.exit() diff --git a/tradedangerous/mfd/saitek/x52pro.py b/tradedangerous/mfd/saitek/x52pro.py index 153ee822..c97203e4 100644 --- a/tradedangerous/mfd/saitek/x52pro.py +++ b/tradedangerous/mfd/saitek/x52pro.py @@ -13,13 +13,14 @@ * Error handling and exceptions """ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals - from tradedangerous.mfd.saitek.directoutput import DirectOutputDevice +import sys +import time + class X52Pro(DirectOutputDevice): - class Page(object): + class Page: _lines = [ str(), str(), str() ] _leds = dict() @@ -45,7 +46,7 @@ def __setitem__(self, key, value): self.device.SetString(self.page_id, key, value) def activate(self): - if self.active == True: + if self.active is True: return self.device.AddPage(self.page_id, self.name, 1) @@ -172,8 +173,6 @@ def finish(self): print("Looping") - import time, sys - loopNo = 0 while True: try: @@ -194,4 +193,3 @@ def finish(self): print(e) x52.finish() sys.exit() - diff --git a/tradedangerous/misc/checkpricebounds.py b/tradedangerous/misc/checkpricebounds.py index 1e926775..08076884 100644 --- a/tradedangerous/misc/checkpricebounds.py +++ b/tradedangerous/misc/checkpricebounds.py @@ -143,7 +143,7 @@ def remediation(item, compare, value): if stations: print() print("Generating", deletePrices) - now = tdb.query("SELECT CURRENT_TIMESTAMP").fetchone()[0]; + now = tdb.query("SELECT CURRENT_TIMESTAMP").fetchone()[0] with open(deletePrices, "w", encoding="utf-8") as fh: print("# Deletions based on {} prices".format( table, @@ -175,8 +175,6 @@ def remediation(item, compare, value): db.commit() def main(): - doDeletions = False - parser = argparse.ArgumentParser( description='Check for prices that are outside reasonable bounds.' ) @@ -284,5 +282,6 @@ def main(): errorFilter=errorFilter, ) + if __name__ == "__main__": main() diff --git a/tradedangerous/misc/clipboard.py b/tradedangerous/misc/clipboard.py index e39689d6..6b8b51ff 100644 --- a/tradedangerous/misc/clipboard.py +++ b/tradedangerous/misc/clipboard.py @@ -8,7 +8,7 @@ try: from tkinter import Tk - class SystemNameClip(object): + class SystemNameClip: """ A cross-platform wrapper for copying system names into the clipboard such that they can be pasted into the E:D @@ -41,10 +41,9 @@ def copy_text(self, text): "Set the environment variable 'NOTK' to disable this warning." ) - class SystemNameClip(object): + class SystemNameClip: """ Dummy implementation when tkinter is not available. """ def copy_text(self, text): pass - diff --git a/tradedangerous/misc/coord64.py b/tradedangerous/misc/coord64.py index 2be1501c..28c552a5 100644 --- a/tradedangerous/misc/coord64.py +++ b/tradedangerous/misc/coord64.py @@ -17,12 +17,13 @@ # Original Author: Oliver "kfsone" Smith # Released under the "use it with attribution" license. -from __future__ import print_function, division import string + alphabet = string.digits + string.ascii_lowercase + string.ascii_uppercase + '_.' precision = 100. + def coord_to_d64(coord): i = int(abs(coord * precision)) diff --git a/tradedangerous/misc/derp-sentinel.py b/tradedangerous/misc/derp-sentinel.py index 7294cb0e..1d16116b 100644 --- a/tradedangerous/misc/derp-sentinel.py +++ b/tradedangerous/misc/derp-sentinel.py @@ -28,8 +28,8 @@ def mutate(text, pos): yield t2 yield from mutate(str(t2), i+len(mutant)) + for name in names: for mutant in mutate(name, 0): if mutant in names: print("{} <-> {}".format(name, mutant)) - diff --git a/tradedangerous/misc/diff-system-csvs.py b/tradedangerous/misc/diff-system-csvs.py index eac57495..7a6f21d3 100644 --- a/tradedangerous/misc/diff-system-csvs.py +++ b/tradedangerous/misc/diff-system-csvs.py @@ -29,8 +29,10 @@ def __str__(self): class Item(namedtuple('Item', [ 'norm', 'name', 'loc' ])): pass + normalizeRe = re.compile('[^A-Za-z0-9\' ]') + def readFile(filename): path = Path(filename) if not path.exists(): @@ -71,6 +73,7 @@ def readFile(filename): return names, locs + oldNames, oldLocs = readFile(sys.argv[1]) newNames, newLocs = readFile(sys.argv[2]) diff --git a/tradedangerous/misc/eddb.py b/tradedangerous/misc/eddb.py index cfb6125f..cc54d535 100644 --- a/tradedangerous/misc/eddb.py +++ b/tradedangerous/misc/eddb.py @@ -11,8 +11,6 @@ Original author: oliver@kfs.org """ -import json -import sys import transfers BASE_URL = "http://eddb.io/archive/v3/" @@ -22,7 +20,7 @@ STATIONS_LITE_JSON = BASE_URL + "stations_lite.json" -class EDDBQuery(object): +class EDDBQuery: """ Base class for querying an EDDB data set and converting the JSON results into an iterable stream. diff --git a/tradedangerous/misc/eddn.py b/tradedangerous/misc/eddn.py index 3019c81f..01202178 100644 --- a/tradedangerous/misc/eddn.py +++ b/tradedangerous/misc/eddn.py @@ -86,12 +86,12 @@ class MarketPrice(namedtuple('MarketPrice', [ pass -class Listener(object): +class Listener: """ Provides an object that will listen to the Elite Dangerous Data Network firehose and capture messages for later consumption. - Rather than individual upates, prices are captured across a window of + Rather than individual updates, prices are captured across a window of between minBatchTime and maxBatchTime. When a new update is received, Rather than returning individual messages, messages are captured across a window of potentially several seconds and returned to the caller in diff --git a/tradedangerous/misc/edsc.py b/tradedangerous/misc/edsc.py index 586ec916..b99d0b3e 100644 --- a/tradedangerous/misc/edsc.py +++ b/tradedangerous/misc/edsc.py @@ -1,17 +1,9 @@ #! /usr/bin/env python3 -from __future__ import absolute_import -from __future__ import with_statement -from __future__ import print_function -from __future__ import division -from __future__ import unicode_literals - -from collections import defaultdict, namedtuple -from urllib.parse import urlencode +from collections import namedtuple from urllib.request import Request, urlopen import json -import os try: import requests @@ -45,7 +37,7 @@ def edsc_log(apiCall, params, jsonData=None, error=None): pass -class EDSCQueryBase(object): +class EDSCQueryBase: """ Base class for creating an EDSC Query class, do not use directly. @@ -116,7 +108,7 @@ class Status(namedtuple('Status', [ ])): pass -class StarSubmissionResult(object): +class StarSubmissionResult: """ Translates a response the json we get back from EDSC when we submit a StarSubmission into something less awful to @@ -228,7 +220,7 @@ def __init__(self, star, response): code = int(ent['status']['statusnum']) msg = ent['status']['msg'] if code in [301, 302, 303, 304]: - if not lhsName in self.distances: + if lhsName not in self.distances: self.distances[lhsName] = {} try: rhsDists = self.distances[rhsName] @@ -238,7 +230,7 @@ def __init__(self, star, response): pass dist = float(ent['dist']) self.distances[lhsName][rhsName] = dist - if not lhsName in self.systems: + if lhsName not in self.systems: self.systems[lhsName] = (code, None) else: if (lhsName,rhsName,code) in errPairs: @@ -335,7 +327,7 @@ def translateCode(self, code): -class StarSubmission(object): +class StarSubmission: baseURL = "http://edstarcoordinator.com/api.asmx/" apiCall = "SubmitDistances" @@ -431,6 +423,7 @@ def submit(self): return innerData + if __name__ == "__main__": print("Requesting recent, non-test, coords-known, cr >= 2 stars") edsq = StarQuery(test=False, confidence=2, known=1) diff --git a/tradedangerous/misc/edsm.py b/tradedangerous/misc/edsm.py index d1b4fd8e..1afca3ab 100644 --- a/tradedangerous/misc/edsm.py +++ b/tradedangerous/misc/edsm.py @@ -5,14 +5,7 @@ uses EDSM - https://www.edsm.net/api """ -from __future__ import absolute_import -from __future__ import with_statement -from __future__ import print_function -from __future__ import division -from __future__ import unicode_literals - import json -import os try: import requests @@ -47,7 +40,7 @@ def edsm_log(apiCall, url, params, jsonData=None, error=None): pass -class EDSMQueryBase(object): +class EDSMQueryBase: """ Base class for creating an EDSM Query class, do not use directly. diff --git a/tradedangerous/misc/importeddbstats.py b/tradedangerous/misc/importeddbstats.py index 8c9a4ca5..4c1d1060 100644 --- a/tradedangerous/misc/importeddbstats.py +++ b/tradedangerous/misc/importeddbstats.py @@ -24,6 +24,7 @@ def matching_stations(): if tdStn: yield tdStn, eddbStn + updateStation = tdb.updateLocalStation bool_trans = { None: '?', 0: 'N', 1: 'Y' } @@ -50,4 +51,4 @@ def matching_stations(): if updates: tdb.getDB().commit() csvexport.exportTableToFile(tdb, tdb.tdenv, "Station") - print("Updated Station.csv: {} updates".format(updates)) \ No newline at end of file + print("Updated Station.csv: {} updates".format(updates)) diff --git a/tradedangerous/misc/prices-json-exp.py b/tradedangerous/misc/prices-json-exp.py index 3a0fcb00..d9851b0b 100644 --- a/tradedangerous/misc/prices-json-exp.py +++ b/tradedangerous/misc/prices-json-exp.py @@ -2,19 +2,21 @@ # Experimental module to generate a JSON version of the .prices file. -# Set to True to allow export of systems that don't have any station data -emptySystems = True -# Set to True to allow export of stations that don't have prices -emptyStations = True - import sqlite3 import json import time import collections import os + +# Set to True to allow export of systems that don't have any station data +emptySystems = True +# Set to True to allow export of stations that don't have prices +emptyStations = True + conn = sqlite3.connect("data/TradeDangerous.db") + def collectItemData(db): """ Builds a flat, array of item names that serves as a table of items. diff --git a/tradedangerous/misc/progress.py b/tradedangerous/misc/progress.py index de5eb04b..48c3676a 100644 --- a/tradedangerous/misc/progress.py +++ b/tradedangerous/misc/progress.py @@ -1,6 +1,6 @@ import sys -class Progress(object): +class Progress: """ Helper class that describes a simple text-based progress bar. """ @@ -68,4 +68,4 @@ def clear(self): if self.textLen: fin = "\r{:{width}}\r".format('', width=self.textLen) sys.stdout.write(fin) - sys.stdout.flush() \ No newline at end of file + sys.stdout.flush() diff --git a/tradedangerous/plugins/__init__.py b/tradedangerous/plugins/__init__.py index efadd412..4c824c0a 100644 --- a/tradedangerous/plugins/__init__.py +++ b/tradedangerous/plugins/__init__.py @@ -10,7 +10,7 @@ class PluginException(Exception): """ -class PluginBase(object): +class PluginBase: """ Base class for plugin implementation. @@ -49,7 +49,7 @@ def __init__(self, tdb, tdenv): else: key, value = opt[:equals], opt[equals+1:] keyName = key.lower() - if not keyName in pluginOptions: + if keyName not in pluginOptions: if keyName == "help": raise SystemExit(self.usage()) diff --git a/tradedangerous/plugins/edapi_plug.py b/tradedangerous/plugins/edapi_plug.py index 0298150d..834d26b5 100644 --- a/tradedangerous/plugins/edapi_plug.py +++ b/tradedangerous/plugins/edapi_plug.py @@ -61,8 +61,7 @@ def log_message(self, format, *args): pass -class OAuthCallbackServer(object): - +class OAuthCallbackServer: def __init__(self, hostname, port, handler): myServer = HTTPServer myServer.callback_code = None @@ -136,8 +135,11 @@ def __init__( # Grab the commander profile self.text = [] self.profile = self.query_capi("/profile") - market = self.query_capi("/market") - shipyard = self.query_capi("/shipyard") + + # kfsone: not sure if there was a reason to query these even tho we didn't + # use the resulting data. + # market = self.query_capi("/market") + # shipyard = self.query_capi("/shipyard") # Grab the market, outfitting and shipyard data if needed portServices = self.profile['lastStarport'].get('services') @@ -503,7 +505,8 @@ def getYNfromObject(obj, key, val = None): def warnAPIResponse(checkName, checkYN): # no warning if unknown - if checkYN == "?": return False + if checkYN == "?": + return False warnText = ( "The station should{s} have a {what}, " "but the API did{d} return one." @@ -582,7 +585,7 @@ def warnAPIResponse(checkName, checkYN): tdenv.WARN("(Fields will be marked with an leading asterisk '*')") askForData = True if ((defStation.lsFromStar == 0) or ("?" in defStation)): - askForData = True + askForData = True newStation = {} for key in defStation._fields: @@ -812,9 +815,7 @@ def run(self): shipCost = {} shipList = [] eddn_ships = [] - if ((station.shipyard == "Y") and - ('ships' in api.profile['lastStarport']) - ): + if ((station.shipyard == "Y") and ('ships' in api.profile['lastStarport'])): if 'shipyard_list' in api.profile['lastStarport']['ships']: if len(api.profile['lastStarport']['ships']['shipyard_list']): for ship in api.profile['lastStarport']['ships']['shipyard_list'].values(): @@ -914,9 +915,7 @@ def run(self): # If a market exists, make the item lists itemList = [] eddn_market = [] - if ((station.market == "Y") and - ('commodities' in api.profile['lastStarport']) - ): + if ((station.market == "Y") and ('commodities' in api.profile['lastStarport'])): for commodity in api.profile['lastStarport']['commodities']: if commodity['categoryname'] in cat_ignore: continue @@ -1046,18 +1045,13 @@ def commodity_int(key): eddn_ships ) - if ((station.outfitting == "Y") and - ('modules' in api.profile['lastStarport'] and - len(api.profile['lastStarport']['modules'])) - ): + if station.outfitting == "Y" and 'modules' in api.profile['lastStarport'] and len(api.profile['lastStarport']['modules']): eddn_modules = [] for module in api.profile['lastStarport']['modules'].values(): # see: https://github.com/EDSM-NET/EDDN/wiki addModule = False if module['name'].startswith(('Hpt_', 'Int_')) or module['name'].find('_Armour_') > 0: - if module.get('sku', None) in ( - None, 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' - ): + if module.get('sku', None) in (None, 'ELITE_HORIZONS_V_PLANETARY_LANDINGS'): if module['name'] != 'Int_PlanetApproachSuite': addModule = True if addModule: diff --git a/tradedangerous/plugins/edcd_plug.py b/tradedangerous/plugins/edcd_plug.py index 32f9d6ed..794544bc 100644 --- a/tradedangerous/plugins/edcd_plug.py +++ b/tradedangerous/plugins/edcd_plug.py @@ -1,8 +1,7 @@ import csv import pathlib -import random -from .. import cache, tradeenv, transfers, csvexport +from .. import cache, transfers, csvexport from ..tradedb import Category, Item from . import PluginException, ImportPluginBase @@ -194,7 +193,7 @@ def process_fdevids_items(self, localPath, tableName): """ More to do for commodities """ - tdb, tdenv = self.tdb, self.tdenv + tdenv = self.tdenv tdenv.NOTE("Processing {}", tableName) itmCount = 0 @@ -207,7 +206,6 @@ def process_fdevids_items(self, localPath, tableName): ) # first line must be the column names columnDefs = next(csvIn) - columnCount = len(columnDefs) tdenv.DEBUG0("columnDefs: {}", columnDefs) @@ -235,7 +233,8 @@ def castToInteger(val): tdenv.NOTE("Import stopped.", checkMe, localPath) else: for lineIn in csvIn: - if not lineIn: continue + if not lineIn: + continue lineNo = csvIn.line_num tdenv.DEBUG0("LINE {}: {}", lineNo, lineIn) diff --git a/tradedangerous/plugins/eddblink_plug.py b/tradedangerous/plugins/eddblink_plug.py index c438ef69..79178db8 100644 --- a/tradedangerous/plugins/eddblink_plug.py +++ b/tradedangerous/plugins/eddblink_plug.py @@ -3,12 +3,10 @@ # a EDDBlink_listener server to update the Database. # ---------------------------------------------------------------- import certifi -import codecs import csv import datetime import json import os -import platform import sqlite3 import ssl import time @@ -16,12 +14,10 @@ from urllib import request from calendar import timegm from pathlib import Path -from importlib import reload -from .. import plugins, cache, csvexport, tradedb, tradeenv, transfers +from .. import plugins, cache, transfers from ..misc import progress as pbar from ..plugins import PluginException -from shutil import copyfile # Constants BASE_URL = os.environ.get('TD_SERVER') or "https://elite.tromador.com/files/" @@ -32,7 +28,7 @@ def request_url(url, headers=None): data = None if headers: data = bytes(json.dumps(headers), encoding="utf-8") - + return request.urlopen(request.Request(url, data=data), context=CONTEXT) @@ -194,25 +190,25 @@ def purgeSystems(self): Purges systems from the System table that do not have any stations claiming to be in them. Keeps table from becoming too large because of fleet carriers moving to unpopulated systems. """ - + self.tdenv.NOTE("Purging Systems with no stations: Start time = {}", self.now()) - + self.execute("PRAGMA foreign_keys = OFF") - + print("Saving systems with stations.... " + str(self.now()) + "\t\t\t\t", end="\r") self.execute("DROP TABLE IF EXISTS System_copy") self.execute("""CREATE TABLE System_copy AS SELECT * FROM System WHERE system_id IN (SELECT system_id FROM Station) """) - + print("Erasing table and reinserting kept systems.... " + str(self.now()) + "\t\t\t\t", end="\r") self.execute("DELETE FROM System") self.execute("INSERT INTO System SELECT * FROM System_copy") - + print("Removing copy.... " + str(self.now()) + "\t\t\t\t", end="\r") self.execute("PRAGMA foreign_keys = ON") self.execute("DROP TABLE IF EXISTS System_copy") - + self.tdenv.NOTE("Finished purging Systems. End time = {}", self.now()) def commit(self): @@ -352,7 +348,7 @@ def run(self): default = True for option in self.options: # if not option in ('force', 'fallback', 'skipvend', 'progbar'): - if not option in ('force', 'skipvend', 'prices'): + if option not in ('force', 'skipvend', 'prices'): default = False if default: self.options["listings"] = True @@ -470,11 +466,11 @@ def run(self): if self.getOption("rare"): if self.downloadFile(self.rareItemPath) or self.getOption("force"): buildCache = True - + if self.getOption("shipvend"): if self.downloadFile(self.shipVendorPath) or self.getOption("force"): buildCache = True - + if self.getOption("upvend"): if self.downloadFile(self.upgradeVendorPath) or self.getOption("force"): buildCache = True diff --git a/tradedangerous/plugins/edmc_batch_plug.py b/tradedangerous/plugins/edmc_batch_plug.py index da662e80..c2a17865 100644 --- a/tradedangerous/plugins/edmc_batch_plug.py +++ b/tradedangerous/plugins/edmc_batch_plug.py @@ -1,8 +1,6 @@ -import os -import re from pathlib import Path -from .. import tradedb, tradeenv, cache, fs +from .. import fs from ..commands.exceptions import CommandLineError from . import PluginException, ImportPluginBase @@ -63,7 +61,7 @@ def sanitize_files(self, files): raise PluginException("EDMC Batch unable to process files with multiple stations. Use normal import.") for s in stations: - if(s in stations_seen): + if s in stations_seen: cur_file = stations_seen[s] # Set the newer file as the one we'll use. stations_seen[s] = self.file_get_newer(cur_file, f) @@ -81,7 +79,7 @@ def set_environment(self, files): fs.ensurefolder(tdenv.tmpDir) batchfile = tdenv.tmpDir / Path(self.BATCH_FILE) if batchfile.exists(): - batchfile.unlink() + batchfile.unlink() # We now have a list of paths. Add all contents to a new file temp_file = open(batchfile, "w") diff --git a/tradedangerous/plugins/journal_plug.py b/tradedangerous/plugins/journal_plug.py index 1dd6f84f..33a796f8 100644 --- a/tradedangerous/plugins/journal_plug.py +++ b/tradedangerous/plugins/journal_plug.py @@ -20,7 +20,7 @@ def snapToGrid32(val): def getYNfromService(obj, key): return "Y" if key in obj else "N" -class JournalStation(object): +class JournalStation: __slots__ = ( 'lsFromStar', 'blackMarket', 'maxPadSize', 'market', 'shipyard', 'outfitting', @@ -303,6 +303,7 @@ def updateJournalSysList(self): if not optShow: try: idJournal = tdb.lookupAdded(self.ADDED_NAME) + tdenv.DEBUG1("idjournal = {}", idJournal) except KeyError: tdenv.WARN("Entry '{}' not found in 'Added' table.", self.ADDED_NAME) tdenv.WARN("Trying to add it myself.") diff --git a/tradedangerous/plugins/netlog_plug.py b/tradedangerous/plugins/netlog_plug.py index a125dc41..942c2e60 100644 --- a/tradedangerous/plugins/netlog_plug.py +++ b/tradedangerous/plugins/netlog_plug.py @@ -3,16 +3,15 @@ import re import pathlib import time as _time -from datetime import datetime, time, timedelta, timezone +from datetime import datetime, timedelta, timezone -from .. import tradedb, tradeenv, csvexport +from .. import csvexport from . import PluginException, ImportPluginBase def snapToGrid32(val): try: val = float(val) - if val < 0: corr = -0.5 - else: corr = +0.5 + corr = -0.5 if val < 0 else +0.5 pos = int(val*32+corr)/32 except: pos = None @@ -138,7 +137,7 @@ def calcSeconds(h=0, m=0, s=0): pass if not headDate: if lineCount > 3: - raise PluginException("Doesn't seem do be a FDEV netLog file") + raise PluginException("Doesn't seem do be a FDEV netLog file") else: statHeader = False if lineCount == 3: @@ -184,6 +183,7 @@ def calcSeconds(h=0, m=0, s=0): if not optShow: try: idNetLog = tdb.lookupAdded(self.ADDED_NAME) + tdenv.DEBUG1("idNetLog = {}", idNetLog) except KeyError: tdenv.WARN("Entry '{}' not found in 'Added' table.", self.ADDED_NAME) tdenv.WARN("Trying to add it myself.") diff --git a/tradedangerous/plugins/spansh_plug.py b/tradedangerous/plugins/spansh_plug.py index 48cfc329..85c16855 100644 --- a/tradedangerous/plugins/spansh_plug.py +++ b/tradedangerous/plugins/spansh_plug.py @@ -1,72 +1,205 @@ -import os +from __future__ import annotations + +from contextlib import contextmanager +from datetime import datetime, timedelta +from pathlib import Path + import sys import time -from datetime import datetime, timedelta +import typing from collections import namedtuple -from pathlib import Path +if sys.version_info.major == 3 and sys.version_info.minor >= 10: + from dataclasses import dataclass +else: + dataclass = False # pylint: disable=invalid-name -import requests -import simdjson +from rich.progress import Progress +import ijson import sqlite3 -from .. import plugins, cache, fs, transfers, csvexport, corrections +from .. import plugins, cache, transfers, csvexport, corrections + +if typing.TYPE_CHECKING: + from typing import Any, Iterable, Optional + from .. tradeenv import TradeEnv SOURCE_URL = 'https://downloads.spansh.co.uk/galaxy_stations.json' STATION_TYPE_MAP = { - 'None' : [0, False], - 'Outpost' : [1, False], - 'Coriolis Starport' : [2, False], - 'Ocellus Starport' : [3, False], - 'Orbis Starport' : [4, False], - 'Planetary Outpost' : [11, True], - 'Planetary Port' : [12, True], - 'Mega ship' : [13, False], - 'Asteroid base' : [14, False], + 'None': [0, False], + 'Outpost': [1, False], + 'Coriolis Starport': [2, False], + 'Ocellus Starport': [3, False], + 'Orbis Starport': [4, False], + 'Planetary Outpost': [11, True], + 'Planetary Port': [12, True], + 'Mega ship': [13, False], + 'Asteroid base': [14, False], 'Drake-Class Carrier': [24, False], # fleet carriers 'Settlement': [25, True], # odyssey settlements } -System = namedtuple('System', 'id,name,pos_x,pos_y,pos_z,modified') -Station = namedtuple('Station', 'id,name,distance,max_pad_size,market,black_market,shipyard,outfitting,rearm,refuel,repair,planetary,type,modified') -Commodity = namedtuple('Commodity', 'id,name,category,demand,supply,sell,buy,modified') +if dataclass: + # Dataclass with slots is considerably cheaper and faster than namedtuple + # but is only reliably introduced in 3.10+ + @dataclass(slots=True) + class System: + id: int + name: str + pos_x: float + pos_y: float + pos_z: float + modified: float | None + + @dataclass(slots=True) + class Station: # pylint: disable=too-many-instance-attributes + id: int + system_id: int + name: str + distance: float + max_pad_size: str + market: str # should be Optional[bool] + black_market: str # should be Optional[bool] + shipyard: str # should be Optional[bool] + outfitting: str # should be Optional[bool] + rearm: str # should be Optional[bool] + refuel: str # should be Optional[bool] + repair: str # should be Optional[bool] + planetary: str # should be Optional[bool] + type: int # station type + modified: float + + + @dataclass(slots=True) + class Commodity: + id: int + name: str + category: str + demand: int + supply: int + sell: int + buy: int + modified: float + +else: + System = namedtuple('System', 'id,name,pos_x,pos_y,pos_z,modified') + Station = namedtuple('Station', + 'id,system_id,name,distance,max_pad_size,' + 'market,black_market,shipyard,outfitting,rearm,refuel,repair,planetary,type,modified') + Commodity = namedtuple('Commodity', 'id,name,category,demand,supply,sell,buy,modified') -class Timing: +class Timing: + """ Helper that provides a context manager for timing code execution. """ + def __init__(self): self.start_ts = None self.end_ts = None - + def __enter__(self): self.start_ts = time.perf_counter() self.end_ts = None return self - + def __exit__(self, *args): self.end_ts = time.perf_counter() - + @property - def elapsed(self): + def elapsed(self) -> Optional[float]: + """ If the timing has finish, calculates the elapsed time. """ if self.start_ts is None: return None return (self.end_ts or time.perf_counter()) - self.start_ts - + @property - def is_finished(self): + def is_finished(self) -> bool: + """ True if the timing has finished. """ return self.end_ts is not None +class Progresser: + """ Encapsulates a potentially transient progress view for a given TradeEnv. """ + def __init__(self, tdenv: 'TradeEnv', title: str, fancy: bool = True, total: Optional[int] = None): + self.started = time.time() + self.tdenv = tdenv + self.progress, self.main_task = None, None + self.title = title + self.fancy = fancy + self.total = total + self.main_task = None + if fancy: + self.progress = Progress(console=self.tdenv.console, transient=True, auto_refresh=True, refresh_per_second=2) + else: + self.progress = None + + def __enter__(self): + if not self.fancy: + self.tdenv.uprint(self.title) + else: + self.progress.start() + self.main_task = self.progress.add_task(self.title, start=True, total=self.total) + return self + + def __exit__(self, *args): + self.progress.stop() + + def update(self, title: str) -> None: + if self.fancy: + self.progress.update(self.main_task, description=title) + else: + self.tdenv.DEBUG1(title) + + @contextmanager + def task(self, title: str, total: Optional[int] = None, parent: Optional[str] = None): + parent = parent or self.main_task + if self.fancy: + task = self.progress.add_task(title, start=True, total=total, parent=parent) + else: + self.tdenv.DEBUG0(title) + task = None + try: + yield task + finally: + if self.fancy: + self.progress.remove_task(task) + if task is not None and parent is not None: + self.progress.update(parent, advance=1) + + def bump(self, task, advance: int = 1, description: Optional[str] = None): + """ Advances the progress of a task by one mark. """ + if self.fancy and task is not None: + self.progress.update(task, advance=advance, description=description) + + +def get_timings(started: float, system_count: int, total_station_count: int, *, min_count: int = 100) -> str: + """ describes how long it is taking to process each system and station """ + elapsed = time.time() - started + timings = "sys=" + if system_count >= min_count: + avg = elapsed / float(system_count) * 1000.0 + timings += f"{avg:5.2f}ms" + else: + timings += "..." + timings += ", stn=" + if total_station_count >= min_count: + avg = elapsed / float(total_station_count) * 1000.0 + timings += f"{avg:5.2f}ms" + else: + timings += "..." + return elapsed, timings + + class ImportPlugin(plugins.ImportPluginBase): """Plugin that downloads data from https://spansh.co.uk/dumps. """ - + pluginOptions = { 'url': f'URL to download galaxy data from (defaults to {SOURCE_URL})', 'file': 'Local filename to import galaxy data from; use "-" to load from stdin', 'maxage': 'Skip all entries older than specified age in days, ex.: maxage=1.5', } - + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.url = self.getOption('url') @@ -74,9 +207,9 @@ def __init__(self, *args, **kwargs): self.maxage = float(self.getOption('maxage')) if self.getOption('maxage') else None assert not (self.url and self.file), 'Provide either url or file, not both' if self.file and (self.file != '-'): - self.file = (Path(self.tdenv.cwDir) / self.file).resolve() - if not (self.tdb.dataPath / Path("TradeDangerous.prices")).exists(): - ri_path = self.tdb.dataPath / Path("RareItem.csv") + self.file = (Path(self.tdenv.cwDir, self.file)).resolve() + if not Path(self.tdb.dataPath, "TradeDangerous.prices").exists(): + ri_path = Path(self.tdb.dataPath, "RareItem.csv") rib_path = ri_path.with_suffix(".tmp") if ri_path.exists(): if rib_path.exists(): @@ -88,52 +221,110 @@ def __init__(self, *args, **kwargs): if rib_path.exists(): rib_path.rename(ri_path) + self.need_commit = False + self.cursor = self.tdb.getDB().cursor() + self.commit_rate = 200 + self.commit_limit = self.commit_rate + self.known_systems = self.load_known_systems() self.known_stations = self.load_known_stations() self.known_commodities = self.load_known_commodities() - - def print(self, *args, **kwargs): - return self.tdenv.uprint(*args, **kwargs) - - def run(self): + + def print(self, *args, **kwargs) -> None: + """ Shortcut to the TradeEnv uprint method. """ + self.tdenv.uprint(*args, **kwargs) + + def commit(self, *, force: bool = False) -> None: + """ Perform a commit if required, but try not to do a crazy amount of committing. """ + if not force and not self.need_commit: + return self.cursor + + if not force and self.commit_limit > 0: + self.commit_limit -= 1 + return self.cursor + + db = self.tdb.getDB() + db.commit() + self.cursor = db.cursor() + + self.commit_limit = self.commit_rate + self.need_commit = False + + def run(self) -> bool: if not self.tdenv.detail: self.print('This will take at least several minutes...') self.print('You can increase verbosity (-v) to get a sense of progress') - with Timing() as timing: + + theme = self.tdenv.theme + BOLD, CLOSE, DIM, ITALIC = theme.bold, theme.CLOSE, theme.dim, theme.italic # pylint: disable=invalid-name + + if not self.file: + url = self.url or SOURCE_URL + self.print(f'Downloading prices from remote URL: {url}') + self.file = Path(self.tdenv.tmpDir, "galaxy_stations.json") + transfers.download(self.tdenv, url, self.file) + self.print(f'Download complete, saved to local file: "{self.file}"') + + + sys_desc = f"Importing {ITALIC}spansh{CLOSE} data" + with Timing() as timing, Progresser(self.tdenv, sys_desc, total=len(self.known_systems)) as progress: system_count = 0 total_station_count = 0 total_commodity_count = 0 - for system, stations in self.data_stream(): - self.ensure_system(system) - station_count = 0 - commodity_count = 0 - for station, commodities in stations: - fq_station_name = f'@{system.name.upper()}/{station.name}' - if self.maxage and (datetime.now() - station.modified) > timedelta(days=self.maxage): - if self.tdenv.detail: - self.print(f' | {fq_station_name:50s} | Skipping station due to age: {(datetime.now() - station.modified) / timedelta (days=1):.2f} days old') - continue - self.ensure_station(system, station) + + age_cutoff = timedelta(days=self.maxage) if self.maxage else None + now = datetime.now() + started = time.time() + + for system, station_iter in self.data_stream(): + upper_sys = system.name.upper() + + elapsed, averages = get_timings(started, system_count, total_station_count) + label = f"{ITALIC}#{system_count:<5d}{CLOSE} {BOLD}{upper_sys:30s}{CLOSE} {DIM}({elapsed:.2f}s, avgs: {averages}){CLOSE}" + stations = list(station_iter) + with progress.task(label, total=len(stations)) as sys_task: + if system.id not in self.known_systems: + self.ensure_system(system, upper_sys) - items = [] - for commodity in commodities: - commodity = self.ensure_commodity(commodity) - result = self.execute("""SELECT modified FROM StationItem - WHERE station_id = ? AND item_id = ?""", - station.id, commodity.id, ).fetchone() - modified = parse_ts(result[0]) if result else None - if modified and commodity.modified <= modified: - # All commodities in a station will have the same modified time, - # so no need to check the rest if the fist is older. + station_count = 0 + commodity_count = 0 + + for station, commodities in stations: + fq_station_name = f'@{upper_sys}/{station.name}' + if age_cutoff and (now - station.modified) > age_cutoff: if self.tdenv.detail: - self.print(f' | {fq_station_name:50s} | Skipping older commodity data') - break - items.append((station.id, commodity.id, commodity.modified, - commodity.sell, commodity.demand, -1, - commodity.buy, commodity.supply, -1, 0)) - if items: - for item in items: - self.execute("""INSERT OR REPLACE INTO StationItem ( + self.print(f' | {fq_station_name:50s} | Skipping station due to age: {now - station.modified}, ts: {station.modified}') + progress.bump(sys_task) + continue + + station_info = self.known_stations.get(station.id) + if not station_info: + self.ensure_station(station) + elif station_info[1] != station.system_id: + self.print(f' | {station.name:50s} | Megaship station moved, updating system') + self.execute("UPDATE Station SET system_id = ? WHERE station_id = ?", station.system_id, station.id, commitable=True) + self.known_stations[station.id] = (station.name, station.system_id) + + items = [] + db_times = dict(self.execute("SELECT item_id, modified FROM StationItem WHERE station_id = ?", station.id)) + + for commodity in commodities: + if commodity.id not in self.known_commodities: + commodity = self.ensure_commodity(commodity) + + db_modified = db_times.get(commodity.id) + modified = parse_ts(db_modified) if db_modified else None + if modified and commodity.modified <= modified: + # All commodities in a station will have the same modified time, + # so no need to check the rest if the fist is older. + if self.tdenv.detail: + self.print(f' | {fq_station_name:50s} | Skipping older commodity data') + break + items.append((station.id, commodity.id, commodity.modified, + commodity.sell, commodity.demand, -1, + commodity.buy, commodity.supply, -1, 0)) + if items: + self.executemany("""INSERT OR REPLACE INTO StationItem ( station_id, item_id, modified, demand_price, demand_units, demand_level, supply_price, supply_units, supply_level, from_live @@ -141,65 +332,97 @@ def run(self): ?, ?, IFNULL(?, CURRENT_TIMESTAMP), ?, ?, ?, ?, ?, ?, ? - )""", *item ) - commodity_count += 1 - self.execute('COMMIT') + )""", items, commitable=True) + commodity_count += len(items) + # Good time to save data and try to keep the transaction small + self.commit() + + if commodity_count: + station_count += 1 + progress.bump(sys_task) - if commodity_count: - station_count += 1 - if station_count: - system_count += 1 - total_station_count += station_count - total_commodity_count += commodity_count - if self.tdenv.detail: - self.print( - f'{system_count:6d} | {system.name.upper():50s} | ' - f'{station_count:3d} st {commodity_count:6d} co' - ) + if station_count: + system_count += 1 + total_station_count += station_count + total_commodity_count += commodity_count + if self.tdenv.detail: + self.print( + f'{system_count:6d} | {upper_sys:50s} | ' + f'{station_count:3d} st {commodity_count:6d} co' + ) + self.commit() + + if system_count % 25 == 1: + avg_stations = total_station_count / (system_count or 1) + progress.update(f"{sys_desc}{DIM} ({total_station_count}:station:, {avg_stations:.1f}per:glowing_star:){CLOSE}") + + self.commit() + + # Need to make sure cached tables are updated, if changes were made + # if self.update_cache: + # for table in [ "Item", "Station", "System" ]: + # _, path = csvexport.exportTableToFile( self.tdb, self.tdenv, table ) - self.execute('COMMIT') self.tdb.close() + # Need to make sure cached tables are updated - for table in [ "Item", "Station", "System" ]: - _, path = csvexport.exportTableToFile( self.tdb, self.tdenv, table ) - + for table in ("Item", "Station", "System", "StationItem"): + # _, path = + csvexport.exportTableToFile(self.tdb, self.tdenv, table) + self.print( f'{timedelta(seconds=int(timing.elapsed))!s} Done ' f'{total_station_count} st {total_commodity_count} co' ) - + return False - + def data_stream(self): - if not self.file: - url = self.url or SOURCE_URL - self.print(f'Downloading prices from remote URL: {url}') - self.file = self.tdenv.tmpDir / Path("galaxy_stations.json") - transfers.download(self.tdenv, url, self.file) - self.print(f'Download complete, saved to local file: {self.file}') - if self.file == '-': self.print('Reading prices from stdin') stream = sys.stdin elif self.file: - self.print(f'Reading prices from local file: {self.file}') + self.print(f'Reading prices from local file: "{self.file}"') stream = open(self.file, 'r', encoding='utf8') return ingest_stream(stream) - + def categorise_commodities(self, commodities): categories = {} for commodity in commodities: categories.setdefault(commodity.category, []).append(commodity) return categories - - def execute(self, query, *params, **kwparams): + + def execute(self, query: str, *params, commitable: bool = False) -> Optional[sqlite3.Cursor]: + """ helper method that performs retriable queries and marks the transaction as needing to commit + if the query is commitable.""" + if commitable: + self.need_commit = True + attempts = 5 + while True: + try: + return self.cursor.execute(query, params) + except sqlite3.OperationalError as ex: + if "no transaction is active" in str(ex): + self.print(f"no transaction for {query}") + return + if not attempts: + raise + attempts -= 1 + self.print(f'Retrying query \'{query}\': {ex!s}') + time.sleep(1) + + def executemany(self, query: str, data: Iterable[Any], *, commitable: bool = False) -> Optional[sqlite3.Cursor]: + """ helper method that performs retriable queries and marks the transaction as needing to commit + if the query is commitable.""" + if commitable: + self.need_commit = True attempts = 5 - cursor = self.tdb.getDB().cursor() while True: try: - return cursor.execute(query, params or kwparams) + return self.cursor.executemany(query, data) except sqlite3.OperationalError as ex: if "no transaction is active" in str(ex): + self.print(f"no transaction for {query}") return if not attempts: raise @@ -207,71 +430,70 @@ def execute(self, query, *params, **kwparams): self.print(f'Retrying query \'{query}\': {ex!s}') time.sleep(1) - def load_known_systems(self): + def load_known_systems(self) -> dict[int, str]: + """ Returns a dictionary of {system_id -> system_name} for all current systems in the database. """ try: - return dict(self.execute('SELECT system_id, name FROM System').fetchall()) - except: - return dict() + return dict(self.cursor.execute('SELECT system_id, name FROM System')) + except Exception as e: # pylint: disable=broad-except + self.print("[purple]:thinking_face:Assuming no system data yet") + self.tdenv.DEBUG0(f"load_known_systems query raised {e}") + return {} - def load_known_stations(self): + def load_known_stations(self) -> dict[int, tuple[str, int]]: + """ Returns a dictionary of {station_id -> (station_name, system_id)} for all current stations in the database. """ try: - return dict(self.execute('SELECT station_id, name FROM Station').fetchall()) - except: - return dict() + return {cols[0]: (cols[1], cols[2]) for cols in self.cursor.execute('SELECT station_id, name, system_id FROM Station')} + except Exception as e: # pylint: disable=broad-except + self.print("[purple]:thinking_face:Assuming no station data yet") + self.tdenv.DEBUG0(f"load_known_stations query raised {e}") + return {} def load_known_commodities(self): + """ Returns a dictionary of {fdev_id -> name} for all current commodities in the database. """ try: - return dict(self.execute('SELECT fdev_id, name FROM Item').fetchall()) - except: - return dict() - - def ensure_system(self, system): - if system.id in self.known_systems: - return + return dict(self.cursor.execute('SELECT fdev_id, name FROM Item')) + except Exception as e: # pylint: disable=broad-except + self.print("[purple]:thinking_face:Assuming no commodity data yet") + self.tdenv.DEBUG0(f"load_known_commodities query raised {e}") + return {} + + def ensure_system(self, system: System, upper_name: str) -> None: + """ Adds a record for a system, and registers the system in the known_systems dict. """ self.execute( ''' INSERT INTO System (system_id, name, pos_x, pos_y, pos_z, modified) VALUES (?, ?, ?, ?, ?, ?) ''', system.id, system.name, system.pos_x, system.pos_y, system.pos_z, system.modified, + commitable=True, ) - self.execute('COMMIT') if self.tdenv.detail > 1: - self.print(f' | {system.name.upper():50s} | Added missing system') + self.print(f' | {upper_name:50s} | Added missing system :glowing_star:') self.known_systems[system.id] = system.name - - def ensure_station(self, system, station): - if station.id in self.known_stations: - system_id = self.execute('SELECT system_id FROM Station WHERE station_id = ?', station.id, ).fetchone()[0] - if system_id != system.id: - self.print(f' | {station.name:50s} | Megaship station moved, updating system') - self.execute("UPDATE Station SET system_id = ? WHERE station_id = ?", system.id, station.id, ) - self.execute('COMMIT') - return + + def ensure_station(self, station: Station) -> None: + """ Adds a record for a station, and registers the station in the known_stations dict. """ self.execute( ''' INSERT INTO Station ( - system_id, - station_id, - name, - ls_from_star, - max_pad_size, - market, - blackmarket, - shipyard, - outfitting, - rearm, - refuel, - repair, + system_id, station_id, name, + ls_from_star, max_pad_size, + market, blackmarket, shipyard, outfitting, + rearm, refuel, repair, planetary, modified, type_id ) VALUES ( - (SELECT system_id FROM System WHERE upper(name) = ?), - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, + ?, ?, + ?, ?, ?, ?, + ?, ?, ?, + ?, + ?, + ? ) ''', - system.name.upper(), + station.system_id, station.id, station.name, station.distance, @@ -286,15 +508,14 @@ def ensure_station(self, system, station): self.bool_yn(station.planetary), station.modified, station.type, + commitable=True, ) - self.execute('COMMIT') if self.tdenv.detail > 1: self.print(f' | {station.name:50s} | Added missing station') - self.known_stations[station.id]= station.name - - def ensure_commodity(self, commodity): - if commodity.id in self.known_commodities: - return commodity + self.known_stations[station.id] = (station.name, station.system_id) + + def ensure_commodity(self, commodity: Commodity): + """ Adds a record for a commodity and registers the commodity in the known_commodities dict. """ self.execute( ''' INSERT INTO Item (item_id, category_id, name, fdev_id) @@ -304,51 +525,51 @@ def ensure_commodity(self, commodity): commodity.category.upper(), corrections.correctItem(commodity.name), commodity.id, + commitable=True, ) # Need to update ui_order - temp = self.execute("""SELECT - name, category_id, fdev_id + temp = self.execute("""SELECT name, category_id, fdev_id, ui_order FROM Item ORDER BY category_id, name """) cat_id = 0 ui_order = 1 self.tdenv.DEBUG0("Updating ui_order data for items.") - for line in temp: - if line[1] != cat_id: + changes = [] + for name, db_cat, fdev_id, db_order in temp: + if db_cat != cat_id: ui_order = 1 - cat_id = line[1] + cat_id = db_cat else: ui_order += 1 - self.execute("""UPDATE Item - set ui_order = ? - WHERE fdev_id = ?""", - ui_order, line[2],) + if ui_order != db_order: + self.tdenv.DEBUG0(f"UI order for {name} ({fdev_id}) needs correction.") + changes += [(ui_order, fdev_id)] + + if changes: + self.executemany( + "UPDATE Item SET ui_order = ? WHERE fdev_id = ?", + changes, + commitable=True + ) - self.execute('COMMIT') - if self.tdenv.detail > 1: - self.print(f' | {commodity.name:50s} | Added missing commodity') self.known_commodities[commodity.id] = commodity.name + return commodity - - def bool_yn(self, value): + + def bool_yn(self, value: Optional[bool]) -> str: + """ translates a ternary (none, true, false) into the ?/Y/N representation """ return '?' if value is None else ('Y' if value else 'N') def ingest_stream(stream): """Ingest a spansh-style galaxy dump, yielding system-level data.""" - line = next(stream) - assert line.rstrip(' \n,') == '[' - for line in stream: - line = line.rstrip().rstrip(',') - if line == ']': - break - system_data = simdjson.Parser().parse(line) + for system_data in ijson.items(stream, 'item', use_float=True): coords = system_data.get('coords', {}) yield ( System( - id = system_data.get('id64'), + id=system_data.get('id64'), name=system_data.get('name', 'Unnamed').strip(), pos_x=coords.get('x', 999999), pos_y=coords.get('y', 999999), @@ -361,6 +582,7 @@ def ingest_stream(stream): def ingest_stations(system_data): """Ingest system-level data, yielding station-level data.""" + sys_id = system_data.get('id64') targets = [system_data, *system_data.get('bodies', ())] for target in targets: for station_data in target.get('stations', ()): @@ -378,9 +600,11 @@ def ingest_stations(system_data): max_pad_size = 'M' elif landing_pads.get('small'): max_pad_size = 'S' + station_type = STATION_TYPE_MAP.get(station_data.get('type')) yield ( Station( - id = station_data.get('id'), + id=station_data.get('id'), + system_id=sys_id, name=station_data.get('name', 'Unnamed').strip(), distance=station_data.get('distanceToArrival', 999999), max_pad_size=max_pad_size, @@ -391,8 +615,8 @@ def ingest_stations(system_data): rearm='Restock' in services, refuel='Refuel' in services, repair='Repair' in services, - planetary=STATION_TYPE_MAP.get(station_data.get('type'))[1] or False, - type=STATION_TYPE_MAP.get(station_data.get('type'))[0] or 0, + planetary=station_type[1] if station_type else False, + type=station_type[0] if station_type else 0, modified=parse_ts(station_data.get('updateTime')), ), ingest_market(market), diff --git a/tradedangerous/prices.py b/tradedangerous/prices.py index 4475406a..f43bfdb3 100644 --- a/tradedangerous/prices.py +++ b/tradedangerous/prices.py @@ -9,20 +9,16 @@ # -------------------------------------------------------------------- # TradeDangerous :: Modules :: Generate TradeDangerous.prices -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals - import sys -import os -import re import sqlite3 -class Element(object): - basic = (1 << 0) - supply = (1 << 1) - timestamp = (1 << 2) - full = (basic | supply | timestamp) - blanks = (1 <<31) +class Element: # TODO: enum? + basic = 1 << 0 + supply = 1 << 1 + timestamp = 1 << 2 + full = basic | supply | timestamp + blanks = 1 <<31 ###################################################################### @@ -42,20 +38,20 @@ def dumpPrices( If file is not none, outputs to the given file handle. """ - withTimes = (elementMask & Element.timestamp) - getBlanks = (elementMask & Element.blanks) + withTimes = elementMask & Element.timestamp + getBlanks = elementMask & Element.blanks conn = sqlite3.connect(str(dbPath)) conn.execute("PRAGMA foreign_keys=ON") cur = conn.cursor() - systems = { ID: name for (ID, name) in cur.execute("SELECT system_id, name FROM System") } + systems = dict(cur.execute("SELECT system_id, name FROM System")) stations = { ID: [ name, systems[sysID] ] for (ID, name, sysID) in cur.execute("SELECT station_id, name, system_id FROM Station") } - categories = { ID: name for (ID, name) in cur.execute("SELECT category_id, name FROM Category") } + categories = dict(cur.execute("SELECT category_id, name FROM Category")) items = { ID: [ name, catID, categories[catID] ] for (ID, name, catID) @@ -118,9 +114,10 @@ def dumpPrices( print(sql) cur.execute(sql) - lastSys, lastStn, lastCat = None, None, None + lastStn, lastCat = None, None - if not file: file = sys.stdout + if not file: + file = sys.stdout if stationID: stationSet = str(stations[stationID]) @@ -226,7 +223,9 @@ def dumpPrices( file.write(output) -if __name__ == "__main__": - import tradedb - tdb = tradedb.TradeDB(load=False) - dumpPrices(tdb.dbPath, elementMask=Element.full) + +# if __name__ == "__main__": +# import tradedb +# +# tdb = tradedb.TradeDB(load=False) +# dumpPrices(tdb.dbPath, elementMask=Element.full) diff --git a/tradedangerous/submit-distances.py b/tradedangerous/submit-distances.py index bc97e78b..e373c7f3 100644 --- a/tradedangerous/submit-distances.py +++ b/tradedangerous/submit-distances.py @@ -12,14 +12,8 @@ # and submit a diff, that'd be greatly appreciated! # -from __future__ import print_function - import argparse -import json -import math import os -import pathlib -import platform import random import re import sys @@ -52,7 +46,7 @@ raise e import pip pip.main(["install", "--upgrade", "requests"]) - import requests + import requests # noqa: F401 standardStars = [ "SOL", @@ -67,7 +61,8 @@ class UsageError(Exception): def __init__(self, argv, error): - self.argv, self.error = argv, error + self.argv, self.error = argv, error + def __str__(self): return error + "\n" + argv.format_usage() diff --git a/tradedangerous/templates/TradeDangerous.sql b/tradedangerous/templates/TradeDangerous.sql index 8d675a55..5c465169 100644 --- a/tradedangerous/templates/TradeDangerous.sql +++ b/tradedangerous/templates/TradeDangerous.sql @@ -1,306 +1,305 @@ --- Definitions for all of the tables used in the SQLite --- cache database. --- --- Source data for TradeDangerous is stored in various --- ".csv" files which provide relatively constant data --- such as star names, the list of known tradeable items, --- etc. --- --- Per-station price data is sourced from ".prices" files --- which are designed to be human readable text that --- closely aproximates the in-game UI. --- --- When the .SQL file or the .CSV files change, TD will --- destroy and rebuild the cache next time it is run. --- --- When the .prices file is changed, only the price data --- is reset. --- --- You can edit this file, if you really need to, if you know --- what you are doing. Or you can use the 'sqlite3' command --- to edit the .db database and then use the '.dump' command --- to regenerate this file, except then you'll lose this nice --- header and I might have to wag my finger at you. --- --- -Oliver - -PRAGMA foreign_keys=ON; -PRAGMA synchronous=OFF; -PRAGMA temp_store=MEMORY; -PRAGMA journal_mode=WAL; -PRAGMA auto_vacuum=INCREMENTAL; - -BEGIN TRANSACTION; - - -CREATE TABLE Added - ( - added_id INTEGER PRIMARY KEY AUTOINCREMENT, - name VARCHAR(40) COLLATE nocase, - - UNIQUE(name) - ); - - -CREATE TABLE System - ( - system_id INTEGER PRIMARY KEY, - name VARCHAR(40) COLLATE nocase, - pos_x DOUBLE NOT NULL, - pos_y DOUBLE NOT NULL, - pos_z DOUBLE NOT NULL, - added_id INTEGER, - modified DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, - - UNIQUE (system_id), - - FOREIGN KEY (added_id) REFERENCES Added(added_id) - ON UPDATE CASCADE - ON DELETE CASCADE - ); -CREATE INDEX idx_system_by_pos ON System (pos_x, pos_y, pos_z, system_id); - - -CREATE TABLE Station - ( - station_id INTEGER PRIMARY KEY, - name VARCHAR(40) COLLATE nocase, - system_id INTEGER NOT NULL, - ls_from_star INTEGER NOT NULL DEFAULT 0 - CHECK (ls_from_star >= 0), - blackmarket TEXT(1) NOT NULL DEFAULT '?' - CHECK (blackmarket IN ('?', 'Y', 'N')), - max_pad_size TEXT(1) NOT NULL DEFAULT '?' - CHECK (max_pad_size IN ('?', 'S', 'M', 'L')), - market TEXT(1) NOT NULL DEFAULT '?' - CHECK (market IN ('?', 'Y', 'N')), - shipyard TEXT(1) NOT NULL DEFAULT '?' - CHECK (shipyard IN ('?', 'Y', 'N')), - modified DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, - outfitting TEXT(1) NOT NULL DEFAULT '?' - CHECK (outfitting IN ('?', 'Y', 'N')), - rearm TEXT(1) NOT NULL DEFAULT '?' - CHECK (rearm IN ('?', 'Y', 'N')), - refuel TEXT(1) NOT NULL DEFAULT '?' - CHECK (refuel IN ('?', 'Y', 'N')), - repair TEXT(1) NOT NULL DEFAULT '?' - CHECK (repair IN ('?', 'Y', 'N')), - planetary TEXT(1) NOT NULL DEFAULT '?' - CHECK (planetary IN ('?', 'Y', 'N')), - type_id INTEGER DEFAULT 0 NOT NULL, - - UNIQUE (station_id), - - FOREIGN KEY (system_id) REFERENCES System(system_id) - ON UPDATE CASCADE - ON DELETE CASCADE - ); -CREATE INDEX idx_station_by_system ON Station (system_id, station_id); -CREATE INDEX idx_station_by_name ON Station (name); - - -CREATE TABLE Ship - ( - ship_id INTEGER PRIMARY KEY, - name VARCHAR(40) COLLATE nocase, - cost INTEGER NOT NULL, - - UNIQUE (ship_id) - ); - - -CREATE TABLE ShipVendor - ( - ship_id INTEGER NOT NULL, - station_id INTEGER NOT NULL, - modified DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, - - PRIMARY KEY (ship_id, station_id), - - FOREIGN KEY (ship_id) REFERENCES Ship(ship_id) - ON UPDATE CASCADE - ON DELETE CASCADE, - FOREIGN KEY (station_id) REFERENCES Station(station_id) - ON UPDATE CASCADE - ON DELETE CASCADE - ) WITHOUT ROWID -; - - -CREATE TABLE Upgrade - ( - upgrade_id INTEGER PRIMARY KEY, - name VARCHAR(40) COLLATE nocase, - weight NUMBER NOT NULL, - cost NUMBER NOT NULL, - - UNIQUE (upgrade_id) - ); - - -CREATE TABLE UpgradeVendor - ( - upgrade_id INTEGER NOT NULL, - station_id INTEGER NOT NULL, - cost INTEGER, - modified DATETIME NOT NULL, - - PRIMARY KEY (upgrade_id, station_id), - - FOREIGN KEY (upgrade_id) REFERENCES Upgrade(upgrade_id) - ON UPDATE CASCADE - ON DELETE CASCADE, - FOREIGN KEY (station_id) REFERENCES Station(station_id) - ON UPDATE CASCADE - ON DELETE CASCADE - ) WITHOUT ROWID -; -CREATE INDEX idx_vendor_by_station_id ON UpgradeVendor (station_id); - -CREATE TABLE RareItem - ( - rare_id INTEGER PRIMARY KEY, - station_id INTEGER NOT NULL, - category_id INTEGER NOT NULL, - name VARCHAR(40) COLLATE nocase, - cost INTEGER, - max_allocation INTEGER, - illegal TEXT(1) NOT NULL DEFAULT '?' - CHECK (illegal IN ('?', 'Y', 'N')), - suppressed TEXT(1) NOT NULL DEFAULT '?' - CHECK (suppressed IN ('?', 'Y', 'N')), - - UNIQUE (name), - - FOREIGN KEY (station_id) REFERENCES Station(station_id) - ON UPDATE CASCADE - ON DELETE CASCADE, - FOREIGN KEY (category_id) REFERENCES Category(category_id) - ON UPDATE CASCADE - ON DELETE CASCADE - ) -; - -CREATE TABLE Category - ( - category_id INTEGER PRIMARY KEY, - name VARCHAR(40) COLLATE nocase, - - UNIQUE (category_id) - ); - - -CREATE TABLE Item - ( - item_id INTEGER PRIMARY KEY, - name VARCHAR(40) COLLATE nocase, - category_id INTEGER NOT NULL, - ui_order INTEGER NOT NULL DEFAULT 0, - avg_price INTEGER, - fdev_id INTEGER, - - UNIQUE (item_id), - UNIQUE (fdev_id), - - FOREIGN KEY (category_id) REFERENCES Category(category_id) - ON UPDATE CASCADE - ON DELETE CASCADE - ); - CREATE INDEX idx_item_by_fdev_id ON Item (fdev_id); - - -CREATE TABLE StationItem - ( - station_id INTEGER NOT NULL, - item_id INTEGER NOT NULL, - demand_price INT NOT NULL, - demand_units INT NOT NULL, - demand_level INT NOT NULL, - supply_price INT NOT NULL, - supply_units INT NOT NULL, - supply_level INT NOT NULL, - modified DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, - from_live INTEGER DEFAULT 0 NOT NULL, - - PRIMARY KEY (station_id, item_id), - FOREIGN KEY (station_id) REFERENCES Station(station_id) - ON UPDATE CASCADE ON DELETE CASCADE, - FOREIGN KEY (item_id) REFERENCES Item(item_id) - ON UPDATE CASCADE ON DELETE CASCADE -); -CREATE INDEX si_mod_stn_itm ON StationItem(modified, station_id, item_id); -CREATE INDEX si_itm_dmdpr ON StationItem(item_id, demand_price) WHERE demand_price > 0; -CREATE INDEX si_itm_suppr ON StationItem(item_id, supply_price) WHERE supply_price > 0; - -CREATE VIEW StationBuying AS -SELECT station_id, - item_id, - demand_price AS price, - demand_units AS units, - demand_level AS level, - modified - FROM StationItem - WHERE demand_price > 0 -; - -CREATE VIEW StationSelling AS -SELECT station_id, - item_id, - supply_price AS price, - supply_units AS units, - supply_level AS level, - modified - FROM StationItem - WHERE supply_price > 0 -; - - --- --- The next two tables (FDevShipyard, FDevOutfitting) are --- used to map the FDev API IDs to data ready for EDDN. --- --- The column names are the same as the header line from --- the EDCD/FDevIDs csv files, so we can just download the --- files (shipyard.csv, outfitting.csv) and save them --- as (FDevShipyard.csv, FDevOutfitting.csv) into the --- data directory. --- --- see https://github.com/EDCD/FDevIDs --- --- The commodity.csv is not needed because TD and EDDN --- are using the same names. --- --- -Bernd - -CREATE TABLE FDevShipyard - ( - id INTEGER NOT NULL, - symbol VARCHAR(40), - name VARCHAR(40) COLLATE nocase, - entitlement VARCHAR(50), - - UNIQUE (id) - ); - - -CREATE TABLE FDevOutfitting - ( - id INTEGER NOT NULL, - symbol VARCHAR(40), - category CHAR(10) - CHECK (category IN ('hardpoint','internal','standard','utility')), - name VARCHAR(40) COLLATE nocase, - mount CHAR(10) - CHECK (mount IN (NULL, 'Fixed','Gimballed','Turreted')), - guidance CHAR(10) - CHECK (guidance IN (NULL, 'Dumbfire','Seeker','Swarm')), - ship VARCHAR(40) COLLATE nocase, - class CHAR(1) NOT NULL, - rating CHAR(1) NOT NULL, - entitlement VARCHAR(50), - - UNIQUE (id) - ); - - -COMMIT; +-- Definitions for all of the tables used in the SQLite +-- cache database. +-- +-- Source data for TradeDangerous is stored in various +-- ".csv" files which provide relatively constant data +-- such as star names, the list of known tradeable items, +-- etc. +-- +-- Per-station price data is sourced from ".prices" files +-- which are designed to be human readable text that +-- closely aproximates the in-game UI. +-- +-- When the .SQL file or the .CSV files change, TD will +-- destroy and rebuild the cache next time it is run. +-- +-- When the .prices file is changed, only the price data +-- is reset. +-- +-- You can edit this file, if you really need to, if you know +-- what you are doing. Or you can use the 'sqlite3' command +-- to edit the .db database and then use the '.dump' command +-- to regenerate this file, except then you'll lose this nice +-- header and I might have to wag my finger at you. +-- +-- -Oliver + +PRAGMA foreign_keys=ON; +PRAGMA synchronous=OFF; +PRAGMA temp_store=MEMORY; +PRAGMA journal_mode=WAL; +PRAGMA auto_vacuum=INCREMENTAL; + +BEGIN TRANSACTION; + + +CREATE TABLE Added + ( + added_id INTEGER PRIMARY KEY AUTOINCREMENT, + name VARCHAR(40) COLLATE nocase, + + UNIQUE(name) + ); + + +CREATE TABLE System + ( + system_id INTEGER PRIMARY KEY, + name VARCHAR(40) COLLATE nocase, + pos_x DOUBLE NOT NULL, + pos_y DOUBLE NOT NULL, + pos_z DOUBLE NOT NULL, + added_id INTEGER, + modified DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + + UNIQUE (system_id), + + FOREIGN KEY (added_id) REFERENCES Added(added_id) + ON UPDATE CASCADE + ON DELETE CASCADE + ); +CREATE INDEX idx_system_by_pos ON System (pos_x, pos_y, pos_z, system_id); + + +CREATE TABLE Station + ( + station_id INTEGER PRIMARY KEY, + name VARCHAR(40) COLLATE nocase, + system_id INTEGER NOT NULL, + ls_from_star INTEGER NOT NULL DEFAULT 0 + CHECK (ls_from_star >= 0), + blackmarket TEXT(1) NOT NULL DEFAULT '?' + CHECK (blackmarket IN ('?', 'Y', 'N')), + max_pad_size TEXT(1) NOT NULL DEFAULT '?' + CHECK (max_pad_size IN ('?', 'S', 'M', 'L')), + market TEXT(1) NOT NULL DEFAULT '?' + CHECK (market IN ('?', 'Y', 'N')), + shipyard TEXT(1) NOT NULL DEFAULT '?' + CHECK (shipyard IN ('?', 'Y', 'N')), + modified DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + outfitting TEXT(1) NOT NULL DEFAULT '?' + CHECK (outfitting IN ('?', 'Y', 'N')), + rearm TEXT(1) NOT NULL DEFAULT '?' + CHECK (rearm IN ('?', 'Y', 'N')), + refuel TEXT(1) NOT NULL DEFAULT '?' + CHECK (refuel IN ('?', 'Y', 'N')), + repair TEXT(1) NOT NULL DEFAULT '?' + CHECK (repair IN ('?', 'Y', 'N')), + planetary TEXT(1) NOT NULL DEFAULT '?' + CHECK (planetary IN ('?', 'Y', 'N')), + type_id INTEGER DEFAULT 0 NOT NULL, + + UNIQUE (station_id), + + FOREIGN KEY (system_id) REFERENCES System(system_id) + ON UPDATE CASCADE + ON DELETE CASCADE + ); +CREATE INDEX idx_station_by_system ON Station (system_id, station_id); +CREATE INDEX idx_station_by_name ON Station (name); + + +CREATE TABLE Ship + ( + ship_id INTEGER PRIMARY KEY, + name VARCHAR(40) COLLATE nocase, + cost INTEGER NOT NULL, + + UNIQUE (ship_id) + ); + + +CREATE TABLE ShipVendor + ( + ship_id INTEGER NOT NULL, + station_id INTEGER NOT NULL, + modified DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + + PRIMARY KEY (ship_id, station_id), + + FOREIGN KEY (ship_id) REFERENCES Ship(ship_id) + ON UPDATE CASCADE + ON DELETE CASCADE, + FOREIGN KEY (station_id) REFERENCES Station(station_id) + ON UPDATE CASCADE + ON DELETE CASCADE + ) WITHOUT ROWID +; + + +CREATE TABLE Upgrade + ( + upgrade_id INTEGER PRIMARY KEY, + name VARCHAR(40) COLLATE nocase, + weight NUMBER NOT NULL, + cost NUMBER NOT NULL, + + UNIQUE (upgrade_id) + ); + + +CREATE TABLE UpgradeVendor + ( + upgrade_id INTEGER NOT NULL, + station_id INTEGER NOT NULL, + cost INTEGER, + modified DATETIME NOT NULL, + + PRIMARY KEY (upgrade_id, station_id), + + FOREIGN KEY (upgrade_id) REFERENCES Upgrade(upgrade_id) + ON UPDATE CASCADE + ON DELETE CASCADE, + FOREIGN KEY (station_id) REFERENCES Station(station_id) + ON UPDATE CASCADE + ON DELETE CASCADE + ) WITHOUT ROWID +; +CREATE INDEX idx_vendor_by_station_id ON UpgradeVendor (station_id); + +CREATE TABLE RareItem + ( + rare_id INTEGER PRIMARY KEY, + station_id INTEGER NOT NULL, + category_id INTEGER NOT NULL, + name VARCHAR(40) COLLATE nocase, + cost INTEGER, + max_allocation INTEGER, + illegal TEXT(1) NOT NULL DEFAULT '?' + CHECK (illegal IN ('?', 'Y', 'N')), + suppressed TEXT(1) NOT NULL DEFAULT '?' + CHECK (suppressed IN ('?', 'Y', 'N')), + + UNIQUE (name), + + FOREIGN KEY (station_id) REFERENCES Station(station_id) + ON UPDATE CASCADE + ON DELETE CASCADE, + FOREIGN KEY (category_id) REFERENCES Category(category_id) + ON UPDATE CASCADE + ON DELETE CASCADE + ) +; + +CREATE TABLE Category + ( + category_id INTEGER PRIMARY KEY, + name VARCHAR(40) COLLATE nocase, + + UNIQUE (category_id) + ); + + +CREATE TABLE Item + ( + item_id INTEGER PRIMARY KEY, + name VARCHAR(40) COLLATE nocase, + category_id INTEGER NOT NULL, + ui_order INTEGER NOT NULL DEFAULT 0, + avg_price INTEGER, + fdev_id INTEGER, + + UNIQUE (item_id), + + FOREIGN KEY (category_id) REFERENCES Category(category_id) + ON UPDATE CASCADE + ON DELETE CASCADE + ); + CREATE INDEX idx_item_by_fdev_id ON Item (fdev_id); + + +CREATE TABLE StationItem + ( + station_id INTEGER NOT NULL, + item_id INTEGER NOT NULL, + demand_price INT NOT NULL, + demand_units INT NOT NULL, + demand_level INT NOT NULL, + supply_price INT NOT NULL, + supply_units INT NOT NULL, + supply_level INT NOT NULL, + modified DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + from_live INTEGER DEFAULT 0 NOT NULL, + + PRIMARY KEY (station_id, item_id), + FOREIGN KEY (station_id) REFERENCES Station(station_id) + ON UPDATE CASCADE ON DELETE CASCADE, + FOREIGN KEY (item_id) REFERENCES Item(item_id) + ON UPDATE CASCADE ON DELETE CASCADE +); +CREATE INDEX si_mod_stn_itm ON StationItem(modified, station_id, item_id); +CREATE INDEX si_itm_dmdpr ON StationItem(item_id, demand_price) WHERE demand_price > 0; +CREATE INDEX si_itm_suppr ON StationItem(item_id, supply_price) WHERE supply_price > 0; + +CREATE VIEW StationBuying AS +SELECT station_id, + item_id, + demand_price AS price, + demand_units AS units, + demand_level AS level, + modified + FROM StationItem + WHERE demand_price > 0 +; + +CREATE VIEW StationSelling AS +SELECT station_id, + item_id, + supply_price AS price, + supply_units AS units, + supply_level AS level, + modified + FROM StationItem + WHERE supply_price > 0 +; + + +-- +-- The next two tables (FDevShipyard, FDevOutfitting) are +-- used to map the FDev API IDs to data ready for EDDN. +-- +-- The column names are the same as the header line from +-- the EDCD/FDevIDs csv files, so we can just download the +-- files (shipyard.csv, outfitting.csv) and save them +-- as (FDevShipyard.csv, FDevOutfitting.csv) into the +-- data directory. +-- +-- see https://github.com/EDCD/FDevIDs +-- +-- The commodity.csv is not needed because TD and EDDN +-- are using the same names. +-- +-- -Bernd + +CREATE TABLE FDevShipyard + ( + id INTEGER NOT NULL, + symbol VARCHAR(40), + name VARCHAR(40) COLLATE nocase, + entitlement VARCHAR(50), + + UNIQUE (id) + ); + + +CREATE TABLE FDevOutfitting + ( + id INTEGER NOT NULL, + symbol VARCHAR(40), + category CHAR(10) + CHECK (category IN ('hardpoint','internal','standard','utility')), + name VARCHAR(40) COLLATE nocase, + mount CHAR(10) + CHECK (mount IN (NULL, 'Fixed','Gimballed','Turreted')), + guidance CHAR(10) + CHECK (guidance IN (NULL, 'Dumbfire','Seeker','Swarm')), + ship VARCHAR(40) COLLATE nocase, + class CHAR(1) NOT NULL, + rating CHAR(1) NOT NULL, + entitlement VARCHAR(50), + + UNIQUE (id) + ); + + +COMMIT; diff --git a/tradedangerous/trade.py b/tradedangerous/trade.py index 5a494b3f..75ade4b7 100644 --- a/tradedangerous/trade.py +++ b/tradedangerous/trade.py @@ -33,12 +33,19 @@ # DEVELOPERS: If you are a programmer who wants TD to do something # cool, please see the TradeDB and TradeCalc modules. TD is designed # to empower other programmers to do cool stuff. +from __future__ import annotations + from tradedangerous import cli -def main(argv = None): - import sys - cli.main(sys.argv) +import sys + + +def main(argv: list[tuple] = None) -> None: + """ Entry point for the TradeDangerous command-line app. """ + if argv is None: + argv = sys.argv + cli.main(argv) + if __name__ == "__main__": - import sys - cli.main(sys.argv) + cli.main(sys.argv) diff --git a/tradedangerous/tradecalc.py b/tradedangerous/tradecalc.py index dbc97f14..ac388b06 100644 --- a/tradedangerous/tradecalc.py +++ b/tradedangerous/tradecalc.py @@ -39,13 +39,12 @@ from collections import defaultdict from collections import namedtuple -from .tradedb import System, Station, Trade, TradeDB, describeAge +from .tradedb import System, Station, Trade, describeAge from .tradedb import Destination from .tradeexcept import TradeException import datetime import locale -import math import os from .misc import progress as pbar import re @@ -132,7 +131,7 @@ def gpt(self): # Classes -class Route(object): +class Route: """ Describes a series of hops where a TradeLoad is picked up at one station, the player travels via 0 or more hyperspace @@ -210,7 +209,7 @@ def __lt__(self, rhs): def __eq__(self, rhs): return self.score == rhs.score and len(self.jumps) == len(rhs.jumps) - def str(self, colorize): + def text(self, colorize) -> str: return "%s -> %s" % (colorize("cyan", self.firstStation.name()), colorize("blue", self.lastStation.name())) def detail(self, tdenv): @@ -220,9 +219,9 @@ def detail(self, tdenv): detail, goalSystem = tdenv.detail, tdenv.goalSystem - colorize = tdenv.colorize if tdenv.color else lambda x, y : y + colorize = tdenv.colorize if tdenv.color else lambda x, y: y - credits = self.startCr + (tdenv.insurance or 0) + credits = self.startCr + (tdenv.insurance or 0) # pylint: disable=redefined-builtin gainCr = 0 route = self.route @@ -232,25 +231,23 @@ def detail(self, tdenv): # around it this morning. def genSubValues(): for hop in hops: - for (tr, qty) in hop[0]: + for tr, _ in hop[0]: yield len(tr.name(detail)) longestNameLen = max(genSubValues()) - text = self.str(colorize) + text = self.text(colorize) if detail >= 1: text += " (score: {:f})".format(self.score) text += "\n" - jumpsFmt = (" Jump {jumps}\n") - cruiseFmt = (" Supercruise to {stn}\n") + jumpsFmt = " Jump {jumps}\n" + cruiseFmt = " Supercruise to {stn}\n" distFmt = None if detail > 1: if detail > 2: text += self.summary() + "\n" if tdenv.maxJumpsPer > 1: - distFmt = ( - " Direct: {dist:0.2f}ly, Trip: {trav:0.2f}ly\n" - ) + distFmt = " Direct: {dist:0.2f}ly, Trip: {trav:0.2f}ly\n" hopFmt = ( " Load from " +colorize("cyan", "{station}") + @@ -443,7 +440,6 @@ def goalDistance(station): ) if dockFmt: stn = route[i + 1] - stnName = stn.name() text += dockFmt.format( station = decorateStation(stn), gain = hopGainCr, @@ -471,7 +467,7 @@ def summary(self): Returns a string giving a short summary of this route. """ - credits, hops, jumps = self.startCr, self.hops, self.jumps + credits, hops, jumps = self.startCr, self.hops, self.jumps # pylint: disable=redefined-builtin ttlGainCr = sum(hop[1] for hop in hops) numJumps = sum( len(hopJumps) - 1 @@ -495,7 +491,7 @@ def summary(self): ) -class TradeCalc(object): +class TradeCalc: """ Container for accessing trade calculations with common properties. """ @@ -546,22 +542,22 @@ def __init__(self, tdb, tdenv = None, fit = None, items = None): if tdenv.avoidItems or items: avoidItemIDs = set(item.ID for item in tdenv.avoidItems) loadItems = items or tdb.itemByID.values() - loadItemIDs = set() + loadItemSet = set() for item in loadItems: ID = item if isinstance(item, int) else item.ID if ID not in avoidItemIDs: - loadItemIDs.add(str(ID)) - if not loadItemIDs: + loadItemSet.add(str(ID)) + if not loadItemSet: raise TradeException("No items to load.") - loadItemIDs = ",".join(str(ID) for ID in loadItemIDs) - wheres.append("(item_id IN ({}))".format(loadItemIDs)) + load_ids = ",".join(str(ID) for ID in loadItemSet) + wheres.append(f"(item_id IN ({load_ids}))") demand = self.stationsBuying = defaultdict(list) supply = self.stationsSelling = defaultdict(list) whereClause = " AND ".join(wheres) or "1" - lastStnID, stnAppend = 0, None + lastStnID = 0 dmdCount, supCount = 0, 0 stmt = """ SELECT station_id, item_id, @@ -589,7 +585,7 @@ def __init__(self, tdb, tdenv = None, fit = None, items = None): raise BadTimestampError( self.tdb, stnID, itmID, timestamp - ) + ) from None if dmdCr > 0: if not minDemand or dmdUnits >= minDemand: dmdAppend((itmID, dmdCr, dmdUnits, dmdLevel, ageS)) @@ -601,7 +597,7 @@ def __init__(self, tdb, tdenv = None, fit = None, items = None): tdenv.DEBUG0("Loaded {} buys, {} sells".format(dmdCount, supCount)) - def bruteForceFit(self, items, credits, capacity, maxUnits): + def bruteForceFit(self, items, credits, capacity, maxUnits): # pylint: disable=redefined-builtin """ Brute-force generation of all possible combinations of items. This is provided to make it easy to validate the results of future @@ -656,7 +652,7 @@ def _fitCombos(offset, cr, cap, level = 1): return _fitCombos(0, credits, capacity) - def fastFit(self, items, credits, capacity, maxUnits): + def fastFit(self, items, credits, capacity, maxUnits): # pylint: disable=redefined-builtin """ Best load calculator using a recursive knapsack-like algorithm to find multiple loads and return the best. @@ -755,7 +751,7 @@ def _fitCombos(offset, cr, cap): # Mark's test run, to spare searching back through the forum posts for it. # python trade.py run --fr="Orang/Bessel Gateway" --cap=720 --cr=11b --ly=24.73 --empty=37.61 --pad=L --hops=2 --jum=3 --loop --summary -vv --progress - def simpleFit(self, items, credits, capacity, maxUnits): + def simpleFit(self, items, credits, capacity, maxUnits): # pylint: disable=redefined-builtin """ Simplistic load calculator: (The item list is sorted with highest profit margin items in front.) @@ -825,7 +821,7 @@ def getTrades(self, srcStation, dstStation, srcSelling = None): buy = getBuy(sell[0], None) if buy: gainCr = buy[1] - sell[1] - if gainCr >= minGainCr and gainCr <= maxGainCr: + if minGainCr <= gainCr <= maxGainCr: addTrade(Trade( itemIdx[sell[0]], sell[1], gainCr, @@ -867,7 +863,7 @@ def getBestHops(self, routes, restrictTo = None): maxLsFromStar = tdenv.maxLs or float('inf') reqBlackMarket = getattr(tdenv, 'blackMarket', False) or False maxAge = getattr(tdenv, 'maxAge') or 0 - credits = tdenv.credits - (getattr(tdenv, 'insurance', 0) or 0) + credits = tdenv.credits - (getattr(tdenv, 'insurance', 0) or 0) # pylint: disable=redefined-builtin fitFunction = self.defaultFit capacity = tdenv.capacity maxUnits = getattr(tdenv, 'limit') or capacity @@ -902,7 +898,7 @@ def getBestHops(self, routes, restrictTo = None): if avoidPlaces: restrictStations = set( stn for stn in restrictStations - if stn not in avoidPlaces and \ + if stn not in avoidPlaces and stn.system not in avoidPlaces ) @@ -940,11 +936,10 @@ def station_iterator(srcStation): for route in routes: if tdenv.progress: prog.increment(1) - tdenv.DEBUG1("Route = {}", route.str(lambda x, y : y)) + tdenv.DEBUG1("Route = {}", route.text(lambda x, y: y)) srcStation = route.lastStation startCr = credits + int(route.gainCr * safetyMargin) - routeJumps = len(route.jumps) srcSelling = getSelling(srcStation.ID, None) srcSelling = tuple( @@ -968,7 +963,8 @@ def station_iterator(srcStation): if unique: uniquePath = route.route elif loopInt: - uniquePath = route.route[-loopInt:-1] + pos_from_end = 0 - loopInt + uniquePath = route.route[pos_from_end:-1] stations = (d for d in station_iterator(srcStation) if (d.station != srcStation) and @@ -986,7 +982,7 @@ def annotate(dest): "destSys {}, destStn {}, jumps {}, distLy {}", dest.system.dbname, dest.station.dbname, - "->".join(jump.str() for jump in dest.via), + "->".join(jump.text() for jump in dest.via), dest.distLy ) return True @@ -1110,7 +1106,7 @@ def sigmoid(x): ) result = [] - for (dst, route, trade, jumps, ly, score) in bestToDest.values(): + for (dst, route, trade, jumps, _, score) in bestToDest.values(): result.append(route.plus(dst, trade, jumps, score)) return result diff --git a/tradedangerous/tradedb.py b/tradedangerous/tradedb.py index 04e0d503..e79860c3 100644 --- a/tradedangerous/tradedb.py +++ b/tradedangerous/tradedb.py @@ -49,41 +49,32 @@ ###################################################################### # Imports +from __future__ import annotations - -from collections import namedtuple, defaultdict -from pathlib import Path -from .tradeenv import TradeEnv -from .tradeexcept import TradeException - -from . import cache, fs +from collections import namedtuple from contextlib import closing +from math import sqrt as math_sqrt +from pathlib import Path import heapq import itertools import locale -import math -import os import re import sqlite3 import sys +import typing + +from .tradeenv import TradeEnv +from .tradeexcept import TradeException +from . import cache, fs + +if typing.TYPE_CHECKING: + from typing import Generator + from typing import Optional, Union -haveNumpy = False -try: - import numpy - import numpy.linalg - haveNumpy = True -except (KeyError, ImportError): - pass -if not haveNumpy: - class numpy(object): - array = False - float32 = False - ascontiguousarray = False - class linalg(object): - norm = False locale.setlocale(locale.LC_ALL, '') + ###################################################################### # Classes @@ -115,22 +106,20 @@ def __str__(self): key(c) for c in anyMatch[0:-1] ) opportunities += " or " + key(anyMatch[-1]) - return '{} "{}" could match {}'.format( - self.lookupType, str(self.searchKey), - opportunities - ) + return f'{self.lookupType} "{self.searchKey}" could match {opportunities}' class SystemNotStationError(TradeException): """ Raised when a station lookup matched a System but could not be automatically reduced to a Station. """ - pass + pass # pylint: disable=unnecessary-pass # (it's not) + ###################################################################### -def makeStellarGridKey(x, y, z): +def make_stellar_grid_key(x: float, y: float, z: float) -> int: """ The Stellar Grid is a map of systems based on their Stellar co-ordinates rounded down to 32lys. This makes it much easier @@ -138,7 +127,8 @@ def makeStellarGridKey(x, y, z): """ return (int(x) >> 5, int(y) >> 5, int(z) >> 5) -class System(object): + +class System: """ Describes a star system which may contain one or more Station objects. @@ -152,65 +142,28 @@ class System(object): '_rangeCache' ) - class RangeCache(object): + class RangeCache: """ Lazily populated cache of neighboring systems. """ def __init__(self): self.systems = [] - self.probedLy = 0. + self.probed_ly = 0. - def __init__( - self, ID, dbname, posX, posY, posZ, addedID, - ary=numpy.array, - nptype=numpy.float32, - ): + def __init__(self, ID, dbname, posX, posY, posZ, addedID) -> None: self.ID = ID self.dbname = dbname self.posX, self.posY, self.posZ = posX, posY, posZ - if ary: - self.pos = ary([posX, posY, posZ], nptype) self.addedID = addedID or 0 self.stations = () self._rangeCache = None @property - def system(self): + def system(self) -> 'System': + """ Returns self for compatibility with the undefined 'Positional' interface. """ return self - def distToSq(self, other): - """ - Returns the square of the distance between two systems. - - It is slightly cheaper to calculate the square of the - distance between two points, so when you are primarily - doing distance checks you can use this less expensive - distance query and only perform a sqrt (** 0.5) on the - distances that fall within your constraint. - - Args: - other: - The other System to measure the distance between. - - Returns: - Distance in light years to the power of 2 (i.e. squared). - - Example: - # Calculate which of [systems] is within 12 ly - # of System "target". - maxLySq = 12 ** 2 # Maximum of 12 ly. - inRange = [] - for sys in systems: - if sys.distToSq(target) <= maxLySq: - inRange.append(sys) - """ - return ( - (self.posX - other.posX) ** 2 + - (self.posY - other.posY) ** 2 + - (self.posZ - other.posZ) ** 2 - ) - - def distanceTo(self, other): + def distanceTo(self, other: 'System') -> float: """ Returns the distance (in ly) between two systems. @@ -226,26 +179,10 @@ def distanceTo(self, other): lhs.distanceTo(rhs), )) """ - return ( - (self.posX - other.posX) ** 2 + - (self.posY - other.posY) ** 2 + - (self.posZ - other.posZ) ** 2 - ) ** 0.5 # fast sqrt - - if haveNumpy: - def all_distances( - self, iterable, - ary=numpy.ascontiguousarray, norm=numpy.linalg.norm, - ): - """ - Takes a list of systems and returns their distances from this system. - """ - return numpy.linalg.norm( - ary([s.pos for s in iterable]) - self.pos, - ord=2, axis=1. - ) + dx, dy, dz = self.posX - other.posX, self.posY - other.posY, self.posZ - other.posZ + return math_sqrt(dx * dx + dy * dy + dz * dz) - def getStation(self, stationName): + def getStation(self, name: str) -> 'Optional[Station]': """ Quick case-insensitive lookup of a station name within the stations in this system. @@ -254,16 +191,17 @@ def getStation(self, stationName): Station() object if a match is found, otherwise None. """ - upperName = stationName.upper() + name = name.upper() for station in self.stations: - if station.dbname.upper() == upperName: + if station.name == name: return station return None - def name(self, detail=0): + def name(self, detail: int = 0) -> str: # pylint: disable=unused-argument + """ Returns the display name for this System.""" return self.dbname - def str(self): + def text(self) -> str: return self.dbname ###################################################################### @@ -278,7 +216,7 @@ class DestinationNode(namedtuple('DestinationNode', [ ])): pass -class Station(object): +class Station: """ Describes a station (trading or otherwise) in a system. @@ -315,8 +253,8 @@ def __init__( self.dataAge = dataAge system.stations = system.stations + (self,) - def name(self, detail=0): - return '%s/%s' % (self.system.dbname, self.dbname) + def name(self, detail: int = 0) -> str: # pylint: disable=unused-argument + return f"{self.system.dbname}/{self.dbname}" def checkPadSize(self, maxPadSize): """ @@ -379,16 +317,16 @@ def checkFleet(self, fleet): Same as checkPlanetary, but for fleet carriers. """ return (not fleet or self.fleet in fleet) - - + + def checkOdyssey(self, odyssey): """ Same as checkPlanetary, but for Odyssey. """ return (not odyssey or self.odyssey in odyssey) - - - def distFromStar(self, addSuffix=False): + + + def distFromStar(self, addSuffix: bool = False) -> str: """ Returns a textual description of the distance from this Station to the parent star. @@ -399,23 +337,20 @@ def distFromStar(self, addSuffix=False): """ ls = self.lsFromStar if not ls: - if addSuffix: - return "Unk" - else: - return '?' + return "Unk" if addSuffix else "?" + + suffix = "ls" if addSuffix else "" + if ls < 1000: - suffix = 'ls' if addSuffix else '' - return '{:n}'.format(ls)+suffix + return f"{ls:n}{suffix}" if ls < 10000: - suffix = 'ls' if addSuffix else '' - return '{:.2f}K'.format(ls / 1000)+suffix + return f"{ls / 1000:.2f}K{suffix}" if ls < 1000000: - suffix = 'ls' if addSuffix else '' - return '{:n}K'.format(int(ls / 1000))+suffix - return '{:.2f}ly'.format(ls / (365*24*60*60)) + return f"{int(ls / 1000):n}K{suffix}" + return f'{ls / (365*24*60*60):.2f}ly' @property - def isTrading(self): + def isTrading(self) -> bool: """ True if the station is thought to be trading. @@ -428,11 +363,11 @@ def isTrading(self): def itemDataAgeStr(self): """ Returns the age in days of item data if present, else "-". """ if self.itemCount and self.dataAge: - return "{:7.2f}".format(self.dataAge) + return f"{self.dataAge:7.2f}" return "-" - def str(self): - return '%s/%s' % (self.system.dbname, self.dbname) + def text(self) -> str: + return f"{self.system.dbname}/{self.dbname}" ###################################################################### @@ -450,7 +385,7 @@ class Ship(namedtuple('Ship', ( stations -- List of Stations ship is sold at. """ - def name(self, detail=0): + def name(self, detail=0): # pylint: disable=unused-argument return self.dbname ###################################################################### @@ -478,13 +413,13 @@ class Category(namedtuple('Category', ( Returns the display name for this Category. """ - def name(self, detail=0): + def name(self, detail=0): # pylint: disable=unused-argument return self.dbname.upper() ###################################################################### -class Item(object): +class Item: """ A product that can be bought/sold in the game. @@ -554,7 +489,7 @@ def name(self, detail=0): ###################################################################### -class TradeDB(object): +class TradeDB: """ Encapsulation for the database layer. @@ -635,7 +570,7 @@ def __init__( load=True, debug=None, ): - self.conn : sqlite3.Connection = None + self.conn: sqlite3.Connection = None self.tradingCount = None tdenv = tdenv or TradeEnv(debug=(debug or 0)) @@ -666,6 +601,18 @@ def __init__( self.avgSelling, self.avgBuying = None, None self.tradingStationCount = 0 + self.addedByID = None + self.systemByID = None + self.systemByName = None + self.stellarGrid = None + self.stationByID = None + self.shipByID = None + self.categoryByID = None + self.itemByID = None + self.itemByName = None + self.itemByFDevID = None + self.rareItemByID = None + self.rareItemByName = None if load: self.reloadCache() @@ -676,22 +623,16 @@ def calculateDistance2(lx, ly, lz, rx, ry, rz): """ Returns the distance in ly between two points. """ - dX = (lx - rx) - dY = (ly - ry) - dZ = (lz - rz) - distSq = (dX ** 2) + (dY ** 2) + (dZ ** 2) - return distSq + dx, dy, dz = lx - rx, ly - ry, lz - rz + return (dx * dx) + (dy * dy) + (dz * dz) @staticmethod def calculateDistance(lx, ly, lz, rx, ry, rz): """ Returns the distance in ly between two points. """ - dX = (lx - rx) - dY = (ly - ry) - dZ = (lz - rz) - distSq = (dX ** 2) + (dY ** 2) + (dZ ** 2) - return distSq ** 0.5 + dx, dy, dz = lx - rx, ly - ry, lz - rz + return math_sqrt((dx * dx) + (dy * dy) + (dz * dz)) ############################################################ # Access to the underlying database. @@ -705,7 +646,7 @@ def getDB(self) -> sqlite3.Connection: conn.execute("PRAGMA synchronous=OFF") conn.execute("PRAGMA temp_store=MEMORY") conn.execute("PRAGMA auto_vacuum=INCREMENTAL") - + conn.create_function('dist2', 6, TradeDB.calculateDistance2) self.conn = conn return conn @@ -912,7 +853,7 @@ def removeLocalSystem( commit=True, ): """ Removes a system and it's stations from the local DB. """ - for stn in self.stations: + for stn in self.stations(): self.removeLocalStation(stn, commit=False) db = self.getDB() db.execute(""" @@ -939,9 +880,9 @@ def __buildStellarGrid(self): Divides the galaxy into a fixed-sized grid allowing us to aggregate small numbers of stars by locality. """ - stellarGrid = self.stellarGrid = dict() + stellarGrid = self.stellarGrid = {} for system in self.systemByID.values(): - key = makeStellarGridKey(system.posX, system.posY, system.posZ) + key = make_stellar_grid_key(system.posX, system.posY, system.posZ) try: grid = stellarGrid[key] except KeyError: @@ -970,9 +911,9 @@ def genStellarGrid(self, system, ly): self.__buildStellarGrid() sysX, sysY, sysZ = system.posX, system.posY, system.posZ - lwrBound = makeStellarGridKey(sysX - ly, sysY - ly, sysZ - ly) - uprBound = makeStellarGridKey(sysX + ly, sysY + ly, sysZ + ly) - lySq = ly ** 2 + lwrBound = make_stellar_grid_key(sysX - ly, sysY - ly, sysZ - ly) + uprBound = make_stellar_grid_key(sysX + ly, sysY + ly, sysZ + ly) + lySq = ly * ly # in 64-bit python, ** invokes a function call making it 4x expensive as *. stellarGrid = self.stellarGrid for x in range(lwrBound[0], uprBound[0]+1): for y in range(lwrBound[1], uprBound[1]+1): @@ -982,17 +923,20 @@ def genStellarGrid(self, system, ly): except KeyError: continue for candidate in grid: - distSq = (candidate.posX - sysX) ** 2 + delta = candidate.posX - sysX + distSq = delta * delta if distSq > lySq: continue - distSq += (candidate.posY - sysY) ** 2 + delta = candidate.posY - sysY + distSq += delta * delta if distSq > lySq: continue - distSq += (candidate.posZ - sysZ) ** 2 + delta = candidate.posZ - sysZ + distSq += delta * delta if distSq > lySq: continue if candidate is not system: - yield candidate, distSq ** 0.5 + yield candidate, math_sqrt(distSq) def genSystemsInRange(self, system, ly, includeSelf=False): """ @@ -1016,32 +960,32 @@ def genSystemsInRange(self, system, ly, includeSelf=False): The distance in lightyears between system and candidate. """ - cache = system._rangeCache - if not cache: - cache = system._rangeCache = System.RangeCache() - cachedSystems = cache.systems + cur_cache = system._rangeCache # pylint: disable=protected-access + if not cur_cache: + cur_cache = system._rangeCache = System.RangeCache() + cached_systems = cur_cache.systems - if ly > cache.probedLy: + if ly > cur_cache.probed_ly: # Consult the database for stars we haven't seen. - cachedSystems = cache.systems = list( + cached_systems = cur_cache.systems = list( self.genStellarGrid(system, ly) ) - cachedSystems.sort(key=lambda ent: ent[1]) - cache.probedLy = ly + cached_systems.sort(key=lambda ent: ent[1]) + cur_cache.probed_ly = ly if includeSelf: yield system, 0. - if cache.probedLy > ly: + if cur_cache.probed_ly > ly: # Cache may contain values outside our view - for sys, dist in cachedSystems: + for candidate, dist in cached_systems: if dist <= ly: - yield sys, dist + yield candidate, dist else: # No need to be conditional inside the loop - yield from cachedSystems + yield from cached_systems - def getRoute(self, origin, dest, maxJumpLy, avoiding=[], stationInterval=0): + def getRoute(self, origin, dest, maxJumpLy, avoiding=None, stationInterval=0): """ Find a shortest route between two systems with an additional constraint that each system be a maximum of maxJumpLy from @@ -1083,6 +1027,9 @@ def getRoute(self, origin, dest, maxJumpLy, avoiding=[], stationInterval=0): """ + if avoiding is None: + avoiding = [] + if isinstance(origin, Station): origin = origin.system if isinstance(dest, Station): @@ -1119,12 +1066,11 @@ def getRoute(self, origin, dest, maxJumpLy, avoiding=[], stationInterval=0): maxPadSize = self.tdenv.padSize if not maxPadSize: - checkStations = lambda system: bool(system.stations()) + def checkStations(system: System) -> bool: # pylint: disable=function-redefined, missing-docstring + return bool(system.stations()) else: - checkStations = lambda system: any( - stn for stn in system.stations - if stn.checkPadSize(maxPadSize) - ) + def checkStations(system: System) -> bool: # pylint: disable=function-redefined, missing-docstring + return any(stn for stn in system.stations if stn.checkPadSize(maxPadSize)) while openSet: weight, curDist, curSysID, stnDist = heappop(openSet) @@ -1182,7 +1128,7 @@ def getRoute(self, origin, dest, maxJumpLy, avoiding=[], stationInterval=0): ############################################################ # Station data. - def stations(self): + def stations(self) -> 'Generator[Station, None, None]': """ Iterate through the list of stations. """ yield from self.stationByID.values() @@ -1211,7 +1157,7 @@ def _loadStations(self): ID, systemID, name, lsFromStar, market, blackMarket, shipyard, maxPadSize, outfitting, rearm, refuel, repair, planetary, type_id - ) in cur : + ) in cur: isFleet = 'Y' if int(type_id) in types['fleet-carrier'] else 'N' isOdyssey = 'Y' if int(type_id) in types['odyssey'] else 'N' station = Station( @@ -1372,7 +1318,7 @@ def updateLocalStation( def _changed(label, old, new): changes.append( - "{}('{}'=>'{}')".format(label, old, new) + f"{label}('{old}'=>'{new}')" ) if name is not None: @@ -1510,7 +1456,7 @@ def lookupPlace(self, name): @system/station """ - if isinstance(name, System) or isinstance(name, Station): + if isinstance(name, (System, Station)): return name slashPos = name.find('/') @@ -1577,7 +1523,8 @@ def lookup(name, candidates): else: anyMatch.append(place) continue - elif subPos > 0: + + if subPos > 0: if placeNameNorm[subPos] == ' ' and \ placeNameNorm[subPos + nameNormLen] == ' ': wordMatch.append(place) @@ -1602,10 +1549,11 @@ def lookup(name, candidates): if sysName: try: - sys = self.systemByName[sysName] - exactMatch = [sys] + system = self.systemByName[sysName] + exactMatch = [system] except KeyError: lookup(sysName, self.systemByID.values()) + if stnName: # Are we considering the name as a station? # (we don't if they type, e,g '@aulin') @@ -1616,10 +1564,10 @@ def lookup(name, candidates): # stations we compare against. Check first if there are # any matches. stationCandidates = [] - for sys in itertools.chain( + for system in itertools.chain( exactMatch, closeMatch, wordMatch, anyMatch ): - stationCandidates += sys.stations + stationCandidates += system.stations # Clear out the candidate lists exactMatch = [] closeMatch = [] @@ -1643,7 +1591,7 @@ def lookup(name, candidates): # Note: this was a TradeException and may need to be again, # but then we need to catch that error in commandenv # when we process avoids - raise LookupError("Unrecognized place: {}".format(name)) + raise LookupError(f"Unrecognized place: {name}") # More than one match raise AmbiguityError( @@ -1661,12 +1609,7 @@ def lookupStation(self, name, system=None): if isinstance(name, System): # When given a system with only one station, return the station. if len(name.stations) != 1: - raise SystemNotStationError( - "System '%s' has %d stations, " - "please specify a station instead." % ( - name.str(), len(name.stations) - ) - ) + raise SystemNotStationError(f"System '{name}' has {len(name.stations)} stations, please specify a station instead.") return name.stations[0] if system: @@ -1675,7 +1618,7 @@ def lookupStation(self, name, system=None): "Station", name, system.stations, key=lambda system: system.dbname) - stationID, station, systemID, system = None, None, None, None + station, system = None, None try: system = TradeDB.listSearch( "System", name, self.systemByID.values(), @@ -1692,9 +1635,7 @@ def lookupStation(self, name, system=None): pass # If neither matched, we have a lookup error. if not (station or system): - raise LookupError( - "'%s' did not match any station or system." % (name) - ) + raise LookupError(f"'{name}' did not match any station or system.") # If we matched both a station and a system, make sure they resovle to # the same station otherwise we have an ambiguity. Some stations have @@ -1711,10 +1652,7 @@ def lookupStation(self, name, system=None): # system otherwise they need to specify a station name. if len(system.stations) != 1: raise SystemNotStationError( - "System '%s' has %d stations, " - "please specify a station instead." % ( - system.name(), len(system.stations) - ) + f"System '{system.name()}' has {len(system.stations)} stations, please specify a station instead." ) return system.stations[0] @@ -1911,7 +1849,7 @@ def _loadItems(self): category = self.categoryByID[categoryID] item = Item( ID, name, category, - '{}/{}'.format(category.dbname, name), + f"{category.dbname}/{name}", avgPrice, fdevID ) itemByID[ID] = item @@ -1945,7 +1883,7 @@ def getAverageSelling(self): Query the database for average selling prices of all items. """ if not self.avgSelling: - self.avgSelling = {itemID: 0 for itemID in self.itemByID.keys()} + self.avgSelling = {itemID: 0 for itemID in self.itemByID} self.avgSelling.update({ ID: int(cr) for ID, cr in self.getDB().execute(""" @@ -1965,7 +1903,7 @@ def getAverageBuying(self): Query the database for average buying prices of all items. """ if not self.avgBuying: - self.avgBuying = {itemID: 0 for itemID in self.itemByID.keys()} + self.avgBuying = {itemID: 0 for itemID in self.itemByID} self.avgBuying.update({ ID: int(cr) for ID, cr in self.getDB().execute(""" @@ -1992,8 +1930,8 @@ def _loadRareItems(self): cost, max_allocation, illegal, suppressed FROM RareItem """ - - + + rareItemByID, rareItemByName = {}, {} stationByID = self.stationByID with closing(self.query(stmt)) as cur: @@ -2005,7 +1943,7 @@ def _loadRareItems(self): category = self.categoryByID[catID] rare = RareItem( ID, station, name, cost, maxAlloc, illegal, suppressed, - category, '{}/{}'.format(category.dbname, name) + category, f"{category.dbname}/{name}" ) rareItemByID[ID] = rareItemByName[name] = rare self.rareItemByID = rareItemByID @@ -2020,7 +1958,6 @@ def _loadRareItems(self): # Price data. def close(self): - self.cur = None if self.conn: self.conn.close() self.conn = None @@ -2036,8 +1973,8 @@ def load(self, maxSystemLinkLy=None): """ self.tdenv.DEBUG1("Loading data") - - + + self._loadAdded() self._loadSystems() @@ -2082,7 +2019,7 @@ class ListSearchMatch(namedtuple('Match', ['key', 'value'])): needle = lookup.translate(normTrans).translate(trimTrans) partialMatch, wordMatch = [], [] # make a regex to match whole words - wordRe = re.compile(r'\b{}\b'.format(lookup), re.IGNORECASE) + wordRe = re.compile(f"\\b{lookup}\\b", re.IGNORECASE) # describe a match for entry in values: entryKey = key(entry) @@ -2113,12 +2050,10 @@ class ListSearchMatch(namedtuple('Match', ['key', 'value'])): ) return partialMatch[0].value # No matches - raise LookupError( - "Error: '%s' doesn't match any %s" % (lookup, listType) - ) + raise LookupError(f"Error: '{lookup}' doesn't match any {listType}") @staticmethod - def normalizedStr(text): + def normalizedStr(text: str) -> str: """ Returns a case folded, sanitized version of 'str' suitable for performing simple and partial matches against. Removes various @@ -2134,7 +2069,7 @@ def normalizedStr(text): ###################################################################### # Assorted helpers -def describeAge(ageInSeconds): +def describeAge(ageInSeconds: Union[float, int]) -> str: """ Turns an age (in seconds) into a text representation. """ @@ -2144,10 +2079,9 @@ def describeAge(ageInSeconds): if hours == 1: return "1 hr" if hours < 48: - return str(hours) + " hrs" + return f"{hours} hrs" days = int(hours / 24) if days < 90: - return str(days) + " days" + return f"{days} days" - months = int(days / 31) - return str(months) + " mths" + return f"{int(days / 31)} mths" diff --git a/tradedangerous/tradeenv.py b/tradedangerous/tradeenv.py index 1f444c55..8cf10ecd 100644 --- a/tradedangerous/tradeenv.py +++ b/tradedangerous/tradeenv.py @@ -6,7 +6,6 @@ import sys import traceback import typing -from contextlib import contextmanager # Import some utilities from the 'rich' library that provide ways to colorize and animate # the console output, along with other useful features. @@ -17,7 +16,8 @@ if typing.TYPE_CHECKING: - from typing import Any, Dict, Iterator + import argparse + from typing import Any, Optional, Union _ROOT = os.path.abspath(os.path.dirname(__file__)) @@ -34,20 +34,142 @@ install_rich_traces(console=STDERR, show_locals=False, extra_lines=1) -class TradeEnv: +class BaseColorTheme: + """ A way to theme the console output colors. The default is none. """ + CLOSE: str = "" # code to stop the last color + dim: str = "" # code to make text dim + bold: str = "" # code to make text bold + italic: str = "" # code to make text italic + # blink: NEVER = "don't you dare" + + # style, label + debug, DEBUG = dim, "#" + note, NOTE = bold, "NOTE" + warn, WARNING = "", "WARNING" + + seq_first: str = "" # the first item in a sequence + seq_last: str = "" # the last item in a sequence + + # Included as examples of how you might use this to manipulate tradecal output. + itm_units: str = "" # the amount of something + itm_name: str = "" # name of that unit + itm_price: str = "" # how much does it cost? + + def render(self, renderable: Any, style: str) -> str: # pragma: no cover, pylint: disable=unused-argument + """ Renders the given printable item with the given style; BaseColorTheme simply uses a string transformation. """ + if isinstance(renderable, str): + return renderable # avoid an allocation + return str(renderable) + + +class BasicRichColorTheme(BaseColorTheme): + """ Provide's 'rich' styling without our own colorization. """ + CLOSE = "[/]" + bold = "[bold]" + dim = "[dim]" + italic = "[italic]" + + # style, label + debug, DEBUG = dim, "#" + note, NOTE = bold, "NOTE" + warn, WARNING = "[orange3]", "WARNING" + + def render(self, renderable: Any, style: str) -> str: # pragma: no cover + style_attr = getattr(self, style, "") + if not style_attr: + return renderable if isinstance(renderable, str) else str(renderable) + return f"{style_attr}{renderable}{self.CLOSE}" + + +class RichColorTheme(BasicRichColorTheme): + """ Demonstrates how you might augment the rich theme with colors to be used fin e.g tradecal. """ + DEBUG = ":spider_web:" + NOTE = ":information_source:" + WARNING = ":warning:" + + # e.g. First station + seq_first = "[cyan]" + # e.g. Last station + seq_last = "[blue]" + + # Included as examples of how you might use this to manipulate tradecal output. + itm_units = "[yellow3]" + itm_name = "[yellow]" + itm_price = "[bold]" + + +class BaseConsoleIOMixin: + """ Base mixin for running output through rich. """ + console: Console + stderr: Console + theme: BaseColorTheme + quiet: bool + + def uprint(self, *args, stderr: bool = False, style: str = None, **kwargs) -> None: + """ + unicode-safe print via console or stderr, with 'rich' markup handling. + """ + console = self.stderr if stderr else self.console + console.print(*args, style=style, **kwargs) + + +class NonUtf8ConsoleIOMixin(BaseConsoleIOMixin): + """ Mixing for running output through rich with UTF8-translation smoothing. """ + def uprint(self, *args, stderr: bool = False, style: str = None, **kwargs) -> None: + """ unicode-handling print: when the stdout stream is not utf-8 supporting, + we do a little extra io work to ensure users don't get confusing unicode + errors. When the output stream *is* utf-8. + + :param stderr: report to stderr instead of stdout + :param style: specify a 'rich' console style to use when the stream supports it + """ + console = self.stderr if stderr else self.console + try: + # Attempt to print; the 'file' argument isn't supported by rich, so we'll + # need to fall-back on old print when someone specifies it. + console.print(*args, style=style, **kwargs) + + except UnicodeEncodeError as e: + # Characters in the output couldn't be translated to unicode. + if not self.quiet: + self.stderr.print( + f"{self.theme.WARN}{self.theme.bold}CAUTION: Your terminal/console couldn't handle some " + "text I tried to print." + ) + if 'EXCEPTIONS' in os.environ: + traceback.print_exc() + else: + self.stderr.print(e) + + # Try to translate each ary into a viable string using utf error-replacement. + components = [ + str(arg) + .encode(TradeEnv.encoding, errors="replace") + .decode(TradeEnv.encoding) + for arg in args + ] + console.print(*components, style=style, **kwargs) + + +# If the console doesn't support UTF8, use the more-complicated implementation. +if str(sys.stdout.encoding).upper() != 'UTF-8': + Utf8SafeConsoleIOMixin = NonUtf8ConsoleIOMixin +else: + Utf8SafeConsoleIOMixin = BaseConsoleIOMixin + + +class TradeEnv(Utf8SafeConsoleIOMixin): """ - Container for a TradeDangerous "environment", which is a collection of operational parameters. + TradeDangerous provides a container for runtime configuration (cli flags, etc) and io operations to + enable normalization of things without having to pass huge sets of arguments. This includes things + like logging and reporting functionality. To print debug lines, use DEBUG, e.g. DEBUG0, which takes a format() string and parameters, e.g. - DEBUG1("hello, {world}{}", "!", world="world") is equivalent to: - if tdenv.debug >= 1: - print("#hello, {world}{}".format( - "!", world="world" - )) + print("#hello, {world}{}".format("!", world="world")) Use "NOTE" to print remarks which can be disabled with -q. """ @@ -57,6 +179,7 @@ class TradeEnv: 'detail': 0, 'quiet': 0, 'color': False, + 'theme': BaseColorTheme(), 'dataDir': os.environ.get('TD_DATA') or os.path.join(os.getcwd(), 'data'), 'csvDir': os.environ.get('TD_CSV') or os.environ.get('TD_DATA') or os.path.join(os.getcwd(), 'data'), 'tmpDir': os.environ.get('TD_TMP') or os.path.join(os.getcwd(), 'tmp'), @@ -67,84 +190,37 @@ class TradeEnv: } encoding = sys.stdout.encoding - - if str(sys.stdout.encoding).upper() != 'UTF-8': - def uprint(self, *args, stderr: bool = False, style: str = None, **kwargs) -> None: - """ unicode-handling print: when the stdout stream is not utf-8 supporting, - we do a little extra io work to ensure users don't get confusing unicode - errors. When the output stream *is* utf-8, tradeenv replaces this method - with a less expensive method. - :param stderr: report to stderr instead of stdout - :param style: specify a 'rich' console style to use when the stream supports it - """ - console = self.stderr if stderr else self.console - try: - # Attempt to print; the 'file' argument isn't spuported by rich, so we'll - # need to fall-back on old print when someone specifies it. - console.print(*args, style=style, **kwargs) - - except UnicodeEncodeError as e: - # Characters in the output couldn't be translated to unicode. - if not self.quiet: - self.stderr.print( - "[orange3][bold]CAUTION: Your terminal/console couldn't handle some " - "text I tried to print." - ) - if 'EXCEPTIONS' in os.environ: - traceback.print_exc() - else: - self.stderr.print(e) - - # Try to translate each ary into a viable stirng using utf error-replacement. - strs = [ - str(arg) - .encode(TradeEnv.encoding, errors="replace") - .decode(TradeEnv.encoding) - for arg in args - ] - console.print(*strs, style=style, **kwargs) - else: - def uprint(self, *args, stderr: bool = False, style: str = None, **kwargs) -> None: - """ unicode-handling print: when the stdout stream is not utf-8 supporting, - this method is replaced with one that tries to provide users better support - when a unicode error appears in the wild. - - :param file: [optional] stream to write to (disables styles/rich support) - :param style: [optional] specify a rich style for the output text - """ - console = self.stderr if stderr else self.console - console.print(*args, style=style, **kwargs) - - - def __init__(self, properties: Optional[Union[argparse.Namespace, Dict]] = None, **kwargs) -> None: + def __init__(self, properties: Optional[Union[argparse.Namespace, dict]] = None, **kwargs) -> None: # Inject the defaults into ourselves in a dict-like way self.__dict__.update(self.defaults) - + # If properties is a namespace, extract the dictionary; otherwise use it as-is if properties and hasattr(properties, '__dict__'): # which arparse.Namespace has properties = properties.__dict__ # Merge into our dictionary self.__dict__.update(properties or {}) - + # Merge the kwargs dictionary self.__dict__.update(kwargs or {}) - + # When debugging has been enabled on startup, enable slightly more # verbose rich backtraces. if self.__dict__['debug']: install_rich_traces(console=STDERR, show_locals=True, extra_lines=2) - + + self.theme = RichColorTheme() if self.__dict__['color'] else BasicRichColorTheme() + def __getattr__(self, key: str) -> Any: """ Return the default for attributes we don't have """ - + # The first time the DEBUG attribute is referenced, register a method for it. if key.startswith("DEBUG"): # Self-assembling DEBUGN functions def __DEBUG_ENABLED(outText, *args, **kwargs): # Give debug output a less contrasted color. - self.console.print(f"[dim]#{outText.format(*args, **kwargs)}[/dim]") + self.console.print(f"{self.theme.debug}{self.theme.DEBUG}{outText.format(*args, **kwargs)}") def __DEBUG_DISABLED(*args, **kwargs): pass @@ -163,8 +239,7 @@ def __DEBUG_DISABLED(*args, **kwargs): def __NOTE_ENABLED(outText, *args, stderr: bool = False, **kwargs): self.uprint( - "NOTE:", str(outText).format(*args, **kwargs), - style="bold", + f"{self.theme.note}{self.theme.NOTE}: {str(outText).format(*args, **kwargs)}", stderr=stderr, ) @@ -183,8 +258,7 @@ def __NOTE_DISABLED(*args, **kwargs): def _WARN_ENABLED(outText, *args, stderr: bool = False, **kwargs): self.uprint( - "WARNING:", str(outText).format(*args, **kwargs), - style="orange3", + f"{self.theme.warn}{self.theme.WARNING}: {str(outText).format(*args, **kwargs)}", stderr=stderr, ) diff --git a/tradedangerous/tradegui.py b/tradedangerous/tradegui.py index 5aea65fc..f2a85a28 100644 --- a/tradedangerous/tradegui.py +++ b/tradedangerous/tradegui.py @@ -15,8 +15,10 @@ from tradedangerous import gui + def main(argv = None): - gui.main() + gui.main() + if __name__ == "__main__": - gui.main() + gui.main() diff --git a/tradedangerous/transfers.py b/tradedangerous/transfers.py index 1975ec96..1f2b40cd 100644 --- a/tradedangerous/transfers.py +++ b/tradedangerous/transfers.py @@ -1,15 +1,12 @@ -from __future__ import absolute_import, with_statement, print_function, division, unicode_literals -from os import getcwd, path from collections import deque from pathlib import Path from .tradeexcept import TradeException import csv import json -import math from .misc import progress as pbar +import platform # noqa: F401 from . import fs -import platform import time import subprocess import sys @@ -22,7 +19,6 @@ def import_requests(): global __requests - global platform if __requests: return __requests @@ -58,9 +54,8 @@ def import_requests(): raise TradeException("Missing package: 'requests'") try: - import pip + import pip # noqa: F401 # pylint: disable=unused-import except ImportError as e: - import platform raise TradeException( "Python 3.4.2 includes a package manager called 'pip', " "except it doesn't appear to be installed on your system:\n" @@ -70,16 +65,15 @@ def import_requests(): # Let's use "The most reliable approach, and the one that is fully supported." # Especially since the old way produces an error for me on Python 3.6: # "AttributeError: 'module' object has no attribute 'main'" - #pip.main(["install", "--upgrade", "requests"]) + # pip.main(["install", "--upgrade", "requests"]) subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'requests']) try: - import requests + import requests # pylint: disable=redefined-outer-name __requests = requests except ImportError as e: raise TradeException( - "The requests module did not install correctly.{}" - .format(extra) + f"The requests module did not install correctly ({e}).{extra}" ) from None return __requests @@ -270,7 +264,7 @@ def get_json_data(url, *, timeout: int = 90): return json.loads(jsData.decode()) -class CSVStream(object): +class CSVStream: """ Provides an iterator that fetches CSV data from a given URL and presents it as an iterable of (columns, values).