diff --git a/README.md b/README.md index eb9e13553d..d3b4b0a8dc 100644 --- a/README.md +++ b/README.md @@ -24,15 +24,16 @@ We created hummingbot to promote **decentralized market-making**: enabling membe | logo | id | name | ver | doc | status | |:---:|:---:|:---:|:---:|:---:|:---:| +| AscendEx | ascend_ex | [AscendEx](https://ascendex.com/en/global-digital-asset-platform) | 1 | [API](https://ascendex.github.io/ascendex-pro-api/#ascendex-pro-api-documentation) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Beaxy | beaxy | [Beaxy](https://beaxy.com/) | 2 | [API](https://beaxyapiv2trading.docs.apiary.io/) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | | Binance | binance | [Binance](https://www.binance.com/) | 3 | [API](https://github.com/binance/binance-spot-api-docs/blob/master/rest-api.md) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Binance US | binance_us | [Binance US](https://www.binance.com/) | 3 | [API](https://github.com/binance-us/binance-official-api-docs/blob/master/rest-api.md) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | | Binance Perpetual | binance_perpetual | [Binance Futures](https://www.binance.com/) | 1 | [API](https://binance-docs.github.io/apidocs/futures/en/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | |Bittrex Global| bittrex | [Bittrex Global](https://global.bittrex.com/) | 3 | [API](https://bittrex.github.io/api/v3) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | | Bitfinex | bitfinex | [Bitfinex](https://www.bitfinex.com/) | 2 | [API](https://docs.bitfinex.com/docs/introduction) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | -| BitMax | bitmax | [BitMax](https://bitmax.io/en/global-digital-asset-platform) | 1 | [API](https://bitmax-exchange.github.io/bitmax-pro-api/#bitmax-pro-api-documentation) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Blocktane | blocktane | [Blocktane](https://blocktane.io/) | 2 | [API](https://blocktane.io/api) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Coinbase Pro | coinbase_pro | [Coinbase Pro](https://pro.coinbase.com/) | * | [API](https://docs.pro.coinbase.com/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | +| CoinZoom | coinzoom | [CoinZoom](https://trade.coinzoom.com/landing) | * | [API](https://api-docs.coinzoom.com/) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | | Crypto.com | crypto_com | [Crypto.com](https://crypto.com/exchange) | 2 | [API](https://exchange-docs.crypto.com/#introduction) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | | DyDx | dydx | [dy/dx](https://dydx.exchange/) | 1 | [API](https://docs.dydx.exchange/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Eterbase | eterbase | [Eterbase](https://www.eterbase.com/) | * | [API](https://developers.eterbase.exchange/?version=latest) |![RED](https://via.placeholder.com/15/f03c15/?text=+) | diff --git a/assets/ascend_ex_logo.png b/assets/ascend_ex_logo.png new file mode 100644 index 0000000000..c30f757cf4 Binary files /dev/null and b/assets/ascend_ex_logo.png differ diff --git a/assets/bitmax_logo.png b/assets/bitmax_logo.png deleted file mode 100644 index 362daeca21..0000000000 Binary files a/assets/bitmax_logo.png and /dev/null differ diff --git a/assets/coinzoom_logo.svg b/assets/coinzoom_logo.svg new file mode 100644 index 0000000000..8184f907e7 --- /dev/null +++ b/assets/coinzoom_logo.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/conf/__init__.py b/conf/__init__.py index 7854c51a99..69e87838fa 100644 --- a/conf/__init__.py +++ b/conf/__init__.py @@ -108,6 +108,11 @@ hitbtc_api_key = os.getenv("HITBTC_API_KEY") hitbtc_secret_key = os.getenv("HITBTC_SECRET_KEY") +# CoinZoom Test +coinzoom_api_key = os.getenv("COINZOOM_API_KEY") +coinzoom_secret_key = os.getenv("COINZOOM_SECRET_KEY") +coinzoom_username = os.getenv("COINZOOM_USERNAME") + # Wallet Tests test_erc20_token_address = os.getenv("TEST_ERC20_TOKEN_ADDRESS") web3_test_private_key_a = os.getenv("TEST_WALLET_PRIVATE_KEY_A") diff --git a/hummingbot/client/command/config_command.py b/hummingbot/client/command/config_command.py index 60f7d8900c..1c72502953 100644 --- a/hummingbot/client/command/config_command.py +++ b/hummingbot/client/command/config_command.py @@ -36,6 +36,7 @@ no_restart_pmm_keys_in_percentage = ["bid_spread", "ask_spread", "order_level_spread", "inventory_target_base_pct"] no_restart_pmm_keys = ["order_amount", "order_levels", "filled_order_delay", "inventory_skew_enabled", "inventory_range_multiplier"] global_configs_to_display = ["0x_active_cancels", + "autofill_import", "kill_switch_enabled", "kill_switch_rate", "telegram_enabled", @@ -44,12 +45,7 @@ "send_error_logs", "script_enabled", "script_file_path", - "manual_gas_price", "ethereum_chain_name", - "ethgasstation_gas_enabled", - "ethgasstation_api_key", - "ethgasstation_gas_level", - "ethgasstation_refresh_time", "gateway_enabled", "gateway_cert_passphrase", "gateway_api_host", @@ -85,7 +81,7 @@ def list_configs(self, # type: HummingbotApplication self._notify("\n".join(lines)) if self.strategy_name is not None: - data = [[cv.key, cv.value] for cv in self.strategy_config_map.values() if not cv.is_secure] + data = [[cv.printable_key or cv.key, cv.value] for cv in self.strategy_config_map.values() if not cv.is_secure] df = pd.DataFrame(data=data, columns=columns) self._notify("\nStrategy Configurations:") lines = [" " + line for line in df.to_string(index=False, max_colwidth=50).split("\n")] @@ -239,9 +235,12 @@ async def inventory_price_prompt( exchange = config_map["exchange"].value market = config_map["market"].value base_asset, quote_asset = market.split("-") - balances = await UserBalances.instance().balances( - exchange, base_asset, quote_asset - ) + if global_config_map["paper_trade_enabled"].value: + balances = global_config_map["paper_trade_account_balance"].value + else: + balances = await UserBalances.instance().balances( + exchange, base_asset, quote_asset + ) if balances.get(base_asset) is None: return diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index eb6d6e7487..4aad717851 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -97,6 +97,9 @@ async def prompt_a_config(self, # type: HummingbotApplication config: ConfigVar, input_value=None, assign_default=True): + if config.key == "inventory_price": + await self.inventory_price_prompt(self.strategy_config_map, input_value) + return if input_value is None: if assign_default: self.app.set_text(parse_config_default_to_text(config)) @@ -105,9 +108,11 @@ async def prompt_a_config(self, # type: HummingbotApplication if self.app.to_stop_config: return + config.value = parse_cvar_value(config, input_value) err_msg = await config.validate(input_value) if err_msg is not None: self._notify(err_msg) + config.value = None await self.prompt_a_config(config) else: config.value = parse_cvar_value(config, input_value) diff --git a/hummingbot/client/command/import_command.py b/hummingbot/client/command/import_command.py index cd9b54cdbd..5799f3a2f4 100644 --- a/hummingbot/client/command/import_command.py +++ b/hummingbot/client/command/import_command.py @@ -1,6 +1,7 @@ import os from hummingbot.core.utils.async_utils import safe_ensure_future +from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.config_helpers import ( update_strategy_config_map_from_file, short_strategy_name, @@ -43,6 +44,9 @@ async def import_config_file(self, # type: HummingbotApplication self.app.change_prompt(prompt=">>> ") if await self.status_check_all(): self._notify("\nEnter \"start\" to start market making.") + autofill_import = global_config_map.get("autofill_import").value + if autofill_import is not None: + self.app.set_text(autofill_import) async def prompt_a_file_name(self # type: HummingbotApplication ): diff --git a/hummingbot/client/command/rate_command.py b/hummingbot/client/command/rate_command.py index a23135c0dd..b784e3c0c3 100644 --- a/hummingbot/client/command/rate_command.py +++ b/hummingbot/client/command/rate_command.py @@ -5,6 +5,7 @@ ) from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.core.rate_oracle.rate_oracle import RateOracle +from hummingbot.client.errors import OracleRateUnavailable s_float_0 = float(0) s_decimal_0 = Decimal("0") @@ -29,14 +30,21 @@ def rate(self, # type: HummingbotApplication async def show_rate(self, # type: HummingbotApplication pair: str, ): + try: + msg = await RateCommand.oracle_rate_msg(pair) + except OracleRateUnavailable: + msg = "Rate is not available." + self._notify(msg) + + @staticmethod + async def oracle_rate_msg(pair: str, + ): pair = pair.upper() - self._notify(f"Source: {RateOracle.source.name}") rate = await RateOracle.rate_async(pair) if rate is None: - self._notify("Rate is not available.") - return + raise OracleRateUnavailable base, quote = pair.split("-") - self._notify(f"1 {base} = {rate} {quote}") + return f"Source: {RateOracle.source.name}\n1 {base} = {rate} {quote}" async def show_token_value(self, # type: HummingbotApplication token: str diff --git a/hummingbot/client/command/start_command.py b/hummingbot/client/command/start_command.py index e8c2e589a3..a8b516d9ec 100644 --- a/hummingbot/client/command/start_command.py +++ b/hummingbot/client/command/start_command.py @@ -17,19 +17,18 @@ from hummingbot.client.config.config_helpers import ( get_strategy_starter_file, ) -from hummingbot.client.settings import ( - STRATEGIES, - SCRIPTS_PATH, - ethereum_gas_station_required, - required_exchanges, -) +import hummingbot.client.settings as settings from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.core.utils.kill_switch import KillSwitch from typing import TYPE_CHECKING from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.core.utils.eth_gas_station_lookup import EthGasStationLookup from hummingbot.script.script_iterator import ScriptIterator from hummingbot.connector.connector_status import get_connector_status, warning_messages +from hummingbot.client.config.config_var import ConfigVar +from hummingbot.client.command.rate_command import RateCommand +from hummingbot.client.config.config_validators import validate_bool +from hummingbot.client.errors import OracleRateUnavailable +from hummingbot.core.rate_oracle.rate_oracle import RateOracle if TYPE_CHECKING: from hummingbot.client.hummingbot_application import HummingbotApplication @@ -59,15 +58,19 @@ def start(self, # type: HummingbotApplication async def start_check(self, # type: HummingbotApplication log_level: Optional[str] = None, restore: Optional[bool] = False): - if self.strategy_task is not None and not self.strategy_task.done(): self._notify('The bot is already running - please run "stop" first') return + if settings.required_rate_oracle: + if not (await self.confirm_oracle_conversion_rate()): + self._notify("The strategy failed to start.") + return + else: + RateOracle.get_instance().start() is_valid = await self.status_check_all(notify_success=False) if not is_valid: return - if self._last_started_strategy_file != self.strategy_file_name: init_logging("hummingbot_logs.yml", override_log_level=log_level.upper() if log_level else None, @@ -85,7 +88,7 @@ async def start_check(self, # type: HummingbotApplication if global_config_map.get("paper_trade_enabled").value: self._notify("\nPaper Trading ON: All orders are simulated, and no real orders are placed.") - for exchange in required_exchanges: + for exchange in settings.required_exchanges: connector = str(exchange) status = get_connector_status(connector) @@ -106,7 +109,7 @@ async def start_market_making(self, # type: HummingbotApplication strategy_name: str, restore: Optional[bool] = False): start_strategy: Callable = get_strategy_starter_file(strategy_name) - if strategy_name in STRATEGIES: + if strategy_name in settings.STRATEGIES: start_strategy(self) else: raise NotImplementedError @@ -133,7 +136,7 @@ async def start_market_making(self, # type: HummingbotApplication script_file = global_config_map["script_file_path"].value folder = dirname(script_file) if folder == "": - script_file = join(SCRIPTS_PATH, script_file) + script_file = join(settings.SCRIPTS_PATH, script_file) if self.strategy_name != "pure_market_making": self._notify("Error: script feature is only available for pure_market_making strategy (for now).") else: @@ -142,9 +145,6 @@ async def start_market_making(self, # type: HummingbotApplication self.clock.add_iterator(self._script_iterator) self._notify(f"Script ({script_file}) started.") - if global_config_map["ethgasstation_gas_enabled"].value and ethereum_gas_station_required(): - EthGasStationLookup.get_instance().start() - self.strategy_task: asyncio.Task = safe_ensure_future(self._run_clock(), loop=self.ev_loop) self._notify(f"\n'{strategy_name}' strategy started.\n" f"Run `status` command to query the progress.") @@ -155,3 +155,30 @@ async def start_market_making(self, # type: HummingbotApplication await self.wait_till_ready(self.kill_switch.start) except Exception as e: self.logger().error(str(e), exc_info=True) + + async def confirm_oracle_conversion_rate(self, # type: HummingbotApplication + ) -> bool: + try: + result = False + self.app.clear_input() + self.placeholder_mode = True + self.app.hide_input = True + for pair in settings.rate_oracle_pairs: + msg = await RateCommand.oracle_rate_msg(pair) + self._notify("\nRate Oracle:\n" + msg) + config = ConfigVar(key="confirm_oracle_use", + type_str="bool", + prompt="Please confirm to proceed if the above oracle source and rates are correct for " + "this strategy (Yes/No) >>> ", + required_if=lambda: True, + validator=lambda v: validate_bool(v)) + await self.prompt_a_config(config) + if config.value: + result = True + except OracleRateUnavailable: + self._notify("Oracle rate is not available.") + finally: + self.placeholder_mode = False + self.app.hide_input = False + self.app.change_prompt(prompt=">>> ") + return result diff --git a/hummingbot/client/command/status_command.py b/hummingbot/client/command/status_command.py index 3cf92785ad..1fb5e69344 100644 --- a/hummingbot/client/command/status_command.py +++ b/hummingbot/client/command/status_command.py @@ -18,7 +18,7 @@ ) from hummingbot.client.config.security import Security from hummingbot.user.user_balances import UserBalances -from hummingbot.client.settings import required_exchanges, ethereum_wallet_required, ethereum_gas_station_required +from hummingbot.client.settings import required_exchanges, ethereum_wallet_required from hummingbot.core.utils.async_utils import safe_ensure_future from typing import TYPE_CHECKING @@ -186,10 +186,6 @@ async def status_check_all(self, # type: HummingbotApplication else: self._notify(" - ETH wallet check: ETH wallet is not connected.") - if ethereum_gas_station_required() and not global_config_map["ethgasstation_gas_enabled"].value: - self._notify(f' - ETH gas station check: Manual gas price is fixed at ' - f'{global_config_map["manual_gas_price"].value}.') - loading_markets: List[ConnectorBase] = [] for market in self.markets.values(): if not market.ready: diff --git a/hummingbot/client/command/stop_command.py b/hummingbot/client/command/stop_command.py index 3848aadea4..3413f2d90f 100644 --- a/hummingbot/client/command/stop_command.py +++ b/hummingbot/client/command/stop_command.py @@ -3,7 +3,7 @@ import threading from typing import TYPE_CHECKING from hummingbot.core.utils.async_utils import safe_ensure_future -from hummingbot.core.utils.eth_gas_station_lookup import EthGasStationLookup +from hummingbot.core.rate_oracle.rate_oracle import RateOracle if TYPE_CHECKING: from hummingbot.client.hummingbot_application import HummingbotApplication @@ -45,8 +45,8 @@ async def stop_loop(self, # type: HummingbotApplication if self.strategy_task is not None and not self.strategy_task.cancelled(): self.strategy_task.cancel() - if EthGasStationLookup.get_instance().started: - EthGasStationLookup.get_instance().stop() + if RateOracle.get_instance().started: + RateOracle.get_instance().stop() if self.markets_recorder is not None: self.markets_recorder.stop() diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index 2f8b491c75..8bc0b2ebcc 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -171,7 +171,7 @@ def get_erc20_token_addresses() -> Dict[str, List]: address_file_path = TOKEN_ADDRESSES_FILE_PATH token_list = {} - resp = requests.get(token_list_url, timeout=3) + resp = requests.get(token_list_url, timeout=1) decoded_resp = resp.json() for token in decoded_resp["tokens"]: diff --git a/hummingbot/client/config/config_var.py b/hummingbot/client/config/config_var.py index 85811bf2fb..6a346653b9 100644 --- a/hummingbot/client/config/config_var.py +++ b/hummingbot/client/config/config_var.py @@ -24,7 +24,8 @@ def __init__(self, # Whether to prompt a user for value when new strategy config file is created prompt_on_new: bool = False, # Whether this is a config var used in connect command - is_connect_key: bool = False): + is_connect_key: bool = False, + printable_key: str = None): self.prompt = prompt self.key = key self.value = None @@ -36,6 +37,7 @@ def __init__(self, self._on_validated = on_validated self.prompt_on_new = prompt_on_new self.is_connect_key = is_connect_key + self.printable_key = printable_key async def get_prompt(self): if inspect.iscoroutinefunction(self.prompt): diff --git a/hummingbot/client/config/global_config_map.py b/hummingbot/client/config/global_config_map.py index 31bd7e566d..7a6c96fa23 100644 --- a/hummingbot/client/config/global_config_map.py +++ b/hummingbot/client/config/global_config_map.py @@ -196,6 +196,14 @@ def global_token_symbol_on_validated(value: str): default=-100, validator=lambda v: validate_decimal(v, Decimal(-100), Decimal(100)), required_if=lambda: global_config_map["kill_switch_enabled"].value), + "autofill_import": + ConfigVar(key="autofill_import", + prompt="What to auto-fill in the prompt after each import command? (start/config) >>> ", + type_str="str", + default=None, + validator=lambda s: None if s in {"start", + "config"} else "Invalid auto-fill prompt.", + required_if=lambda: False), "telegram_enabled": ConfigVar(key="telegram_enabled", prompt="Would you like to enable telegram? >>> ", @@ -290,32 +298,6 @@ def global_token_symbol_on_validated(value: str): type_str="decimal", validator=lambda v: validate_decimal(v, Decimal(0), inclusive=False), default=50), - "ethgasstation_gas_enabled": - ConfigVar(key="ethgasstation_gas_enabled", - prompt="Do you want to enable Ethereum gas station price lookup? >>> ", - required_if=lambda: False, - type_str="bool", - validator=validate_bool, - default=False), - "ethgasstation_api_key": - ConfigVar(key="ethgasstation_api_key", - prompt="Enter API key for defipulse.com gas station API >>> ", - required_if=lambda: global_config_map["ethgasstation_gas_enabled"].value, - type_str="str"), - "ethgasstation_gas_level": - ConfigVar(key="ethgasstation_gas_level", - prompt="Enter gas level you want to use for Ethereum transactions (fast, fastest, safeLow, average) " - ">>> ", - required_if=lambda: global_config_map["ethgasstation_gas_enabled"].value, - type_str="str", - validator=lambda s: None if s in {"fast", "fastest", "safeLow", "average"} - else "Invalid gas level."), - "ethgasstation_refresh_time": - ConfigVar(key="ethgasstation_refresh_time", - prompt="Enter refresh time for Ethereum gas price lookup (in seconds) >>> ", - required_if=lambda: global_config_map["ethgasstation_gas_enabled"].value, - type_str="int", - default=120), "gateway_api_host": ConfigVar(key="gateway_api_host", prompt=None, @@ -361,11 +343,11 @@ def global_token_symbol_on_validated(value: str): default=RateOracleSource.binance.name), "global_token": ConfigVar(key="global_token", - prompt="What is your default display token? (e.g. USDT,USD,EUR) >>> ", + prompt="What is your default display token? (e.g. USD,EUR,BTC) >>> ", type_str="str", required_if=lambda: False, on_validated=global_token_on_validated, - default="USDT"), + default="USD"), "global_token_symbol": ConfigVar(key="global_token_symbol", prompt="What is your default display token symbol? (e.g. $,€) >>> ", diff --git a/hummingbot/client/errors.py b/hummingbot/client/errors.py index 54b19a407c..22e25d2d1f 100644 --- a/hummingbot/client/errors.py +++ b/hummingbot/client/errors.py @@ -7,3 +7,7 @@ class InvalidCommandError(Exception): class ArgumentParserError(Exception): pass + + +class OracleRateUnavailable(Exception): + pass diff --git a/hummingbot/client/hummingbot_application.py b/hummingbot/client/hummingbot_application.py index 7d025e2d68..f777132f21 100644 --- a/hummingbot/client/hummingbot_application.py +++ b/hummingbot/client/hummingbot_application.py @@ -21,7 +21,6 @@ from hummingbot.client.errors import InvalidCommandError, ArgumentParserError from hummingbot.client.config.global_config_map import global_config_map, using_wallet from hummingbot.client.config.config_helpers import ( - get_erc20_token_addresses, get_strategy_config_map, get_connector_class, get_eth_wallet_private_key, @@ -194,10 +193,14 @@ def _initialize_market_assets(market_name: str, trading_pairs: List[str]) -> Lis return market_trading_pairs def _initialize_wallet(self, token_trading_pairs: List[str]): + # Todo: This function should be removed as it's currently not used by current working connectors + if not using_wallet(): return - if not self.token_list: - self.token_list = get_erc20_token_addresses() + # Commented this out for now since get_erc20_token_addresses uses blocking call + + # if not self.token_list: + # self.token_list = get_erc20_token_addresses() ethereum_wallet = global_config_map.get("ethereum_wallet").value private_key = Security._private_keys[ethereum_wallet] diff --git a/hummingbot/client/settings.py b/hummingbot/client/settings.py index c832bcc7e4..19b12c058d 100644 --- a/hummingbot/client/settings.py +++ b/hummingbot/client/settings.py @@ -15,6 +15,9 @@ # Global variables required_exchanges: List[str] = [] requried_connector_trading_pairs: Dict[str, List[str]] = {} +# Set these two variables if a strategy uses oracle for rate conversion +required_rate_oracle: bool = False +rate_oracle_pairs: List[str] = [] # Global static values KEYFILE_PREFIX = "key_file_" diff --git a/hummingbot/client/ui/completer.py b/hummingbot/client/ui/completer.py index ba850a4bf1..bdb7ae66fa 100644 --- a/hummingbot/client/ui/completer.py +++ b/hummingbot/client/ui/completer.py @@ -68,11 +68,12 @@ def get_subcommand_completer(self, first_word: str) -> Completer: @property def _trading_pair_completer(self) -> Completer: trading_pair_fetcher = TradingPairFetcher.get_instance() + market = "" for exchange in sorted(list(CONNECTOR_SETTINGS.keys()), key=len, reverse=True): if exchange in self.prompt_text: market = exchange break - trading_pairs = trading_pair_fetcher.trading_pairs.get(market, []) if trading_pair_fetcher.ready else [] + trading_pairs = trading_pair_fetcher.trading_pairs.get(market, []) if trading_pair_fetcher.ready and market else [] return WordCompleter(trading_pairs, ignore_case=True, sentence=True) @property diff --git a/hummingbot/connector/connector/balancer/balancer_connector.py b/hummingbot/connector/connector/balancer/balancer_connector.py index e1bc1e131d..179c6b2e8f 100644 --- a/hummingbot/connector/connector/balancer/balancer_connector.py +++ b/hummingbot/connector/connector/balancer/balancer_connector.py @@ -7,7 +7,6 @@ import time import ssl import copy -import itertools as it from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL from hummingbot.core.utils import async_ttl_cache from hummingbot.core.network_iterator import NetworkStatus @@ -31,9 +30,9 @@ from hummingbot.connector.connector_base import ConnectorBase from hummingbot.connector.connector.balancer.balancer_in_flight_order import BalancerInFlightOrder from hummingbot.client.settings import GATEAWAY_CA_CERT_PATH, GATEAWAY_CLIENT_CERT_PATH, GATEAWAY_CLIENT_KEY_PATH -from hummingbot.core.utils.eth_gas_station_lookup import get_gas_price from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.client.config.config_helpers import get_erc20_token_addresses +from hummingbot.core.utils.ethereum import check_transaction_exceptions, fetch_trading_pairs +from hummingbot.client.config.fee_overrides_config_map import fee_overrides_config_map s_logger = None s_decimal_0 = Decimal("0") @@ -71,12 +70,9 @@ def __init__(self, """ super().__init__() self._trading_pairs = trading_pairs - tokens = set() + self._tokens = set() for trading_pair in trading_pairs: - tokens.update(set(trading_pair.split("-"))) - self._erc_20_token_list = self.token_list() - self._token_addresses = {t: l[0] for t, l in self._erc_20_token_list.items() if t in tokens} - self._token_decimals = {t: l[1] for t, l in self._erc_20_token_list.items() if t in tokens} + self._tokens.update(set(trading_pair.split("-"))) self._wallet_private_key = wallet_private_key self._ethereum_rpc_url = ethereum_rpc_url self._trading_required = trading_required @@ -84,10 +80,13 @@ def __init__(self, self._shared_client = None self._last_poll_timestamp = 0.0 self._last_balance_poll_timestamp = time.time() + self._last_est_gas_cost_reported = 0 self._in_flight_orders = {} self._allowances = {} self._status_polling_task = None self._auto_approve_task = None + self._initiate_pool_task = None + self._initiate_pool_status = None self._real_time_balance_update = False self._max_swaps = global_config_map['balancer_max_swaps'].value self._poll_notifier = None @@ -96,17 +95,9 @@ def __init__(self, def name(self): return "balancer" - @staticmethod - def token_list(): - return get_erc20_token_addresses() - @staticmethod async def fetch_trading_pairs() -> List[str]: - token_list = BalancerConnector.token_list() - trading_pairs = [] - for base, quote in it.permutations(token_list.keys(), 2): - trading_pairs.append(f"{base}-{quote}") - return trading_pairs + return await fetch_trading_pairs() @property def limit_orders(self) -> List[LimitOrder]: @@ -115,6 +106,26 @@ def limit_orders(self) -> List[LimitOrder]: for in_flight_order in self._in_flight_orders.values() ] + async def initiate_pool(self) -> str: + """ + Initiate connector and cache pools + """ + try: + self.logger().info(f"Initializing Balancer connector and caching pools for {self._trading_pairs}.") + resp = await self._api_request("get", "eth/balancer/start", + {"pairs": json.dumps(self._trading_pairs)}) + status = bool(str(resp["success"])) + if bool(str(resp["success"])): + self._initiate_pool_status = status + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().network( + f"Error initializing {self._trading_pairs} swap pools", + exc_info=True, + app_warning_msg=str(e) + ) + async def auto_approve(self): """ Automatically approves Balancer contract as a spender for token in trading pairs. @@ -138,9 +149,7 @@ async def approve_balancer_spender(self, token_symbol: str) -> Decimal: """ resp = await self._api_request("post", "eth/approve", - {"tokenAddress": self._token_addresses[token_symbol], - "gasPrice": str(get_gas_price()), - "decimals": self._token_decimals[token_symbol], # if not supplied, gateway would treat it eth-like with 18 decimals + {"token": token_symbol, "connector": self.name}) amount_approved = Decimal(str(resp["amount"])) if amount_approved > 0: @@ -155,12 +164,11 @@ async def get_allowances(self) -> Dict[str, Decimal]: :return: A dictionary of token and its allowance (how much Balancer can spend). """ ret_val = {} - resp = await self._api_request("post", "eth/allowances-2", - {"tokenAddressList": ("".join([tok + "," for tok in self._token_addresses.values()])).rstrip(","), - "tokenDecimalList": ("".join([str(dec) + "," for dec in self._token_decimals.values()])).rstrip(","), + resp = await self._api_request("post", "eth/allowances", + {"tokenList": "[" + (",".join(['"' + t + '"' for t in self._tokens])) + "]", "connector": self.name}) - for address, amount in resp["approvals"].items(): - ret_val[self.get_token(address)] = Decimal(str(amount)) + for token, amount in resp["approvals"].items(): + ret_val[token] = Decimal(str(amount)) return ret_val @async_ttl_cache(ttl=5, maxsize=10) @@ -177,15 +185,44 @@ async def get_quote_price(self, trading_pair: str, is_buy: bool, amount: Decimal base, quote = trading_pair.split("-") side = "buy" if is_buy else "sell" resp = await self._api_request("post", - f"balancer/{side}-price", - {"base": self._token_addresses[base], - "quote": self._token_addresses[quote], + "eth/balancer/price", + {"base": base, + "quote": quote, "amount": amount, - "base_decimals": self._token_decimals[base], - "quote_decimals": self._token_decimals[quote], - "maxSwaps": self._max_swaps}) - if resp["price"] is not None: - return Decimal(str(resp["price"])) + "side": side.upper()}) + required_items = ["price", "gasLimit", "gasPrice", "gasCost"] + if any(item not in resp.keys() for item in required_items): + if "info" in resp.keys(): + self.logger().info(f"Unable to get price. {resp['info']}") + else: + self.logger().info(f"Missing data from price result. Incomplete return result for ({resp.keys()})") + else: + gas_limit = resp["gasLimit"] + gas_price = resp["gasPrice"] + gas_cost = resp["gasCost"] + price = resp["price"] + account_standing = { + "allowances": self._allowances, + "balances": self._account_balances, + "base": base, + "quote": quote, + "amount": amount, + "side": side, + "gas_limit": gas_limit, + "gas_price": gas_price, + "gas_cost": gas_cost, + "price": price, + "swaps": len(resp["swaps"]) + } + exceptions = check_transaction_exceptions(account_standing) + for index in range(len(exceptions)): + self.logger().info(f"Warning! [{index+1}/{len(exceptions)}] {side} order - {exceptions[index]}") + + if price is not None and len(exceptions) == 0: + # TODO standardize quote price object to include price, fee, token, is fee part of quote. + fee_overrides_config_map["balancer_maker_fee_amount"].value = Decimal(str(gas_cost)) + fee_overrides_config_map["balancer_taker_fee_amount"].value = Decimal(str(gas_cost)) + return Decimal(str(price)) except asyncio.CancelledError: raise except Exception as e: @@ -255,24 +292,28 @@ async def _create_order(self, amount = self.quantize_order_amount(trading_pair, amount) price = self.quantize_order_price(trading_pair, price) base, quote = trading_pair.split("-") - gas_price = get_gas_price() - api_params = {"base": self._token_addresses[base], - "quote": self._token_addresses[quote], + api_params = {"base": base, + "quote": quote, + "side": trade_type.name.upper(), "amount": str(amount), - "maxPrice": str(price), - "maxSwaps": str(self._max_swaps), - "gasPrice": str(gas_price), - "base_decimals": self._token_decimals[base], - "quote_decimals": self._token_decimals[quote], + "limitPrice": str(price), } - self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, gas_price) try: - order_result = await self._api_request("post", f"balancer/{trade_type.name.lower()}", api_params) + order_result = await self._api_request("post", "eth/balancer/trade", api_params) hash = order_result.get("txHash") + gas_price = order_result.get("gasPrice") + gas_limit = order_result.get("gasLimit") + gas_cost = order_result.get("gasCost") + self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, gas_price) tracked_order = self._in_flight_orders.get(order_id) + + # update onchain balance + await self._update_balances() + if tracked_order is not None: self.logger().info(f"Created {trade_type.name} order {order_id} txHash: {hash} " - f"for {amount} {trading_pair}.") + f"for {amount} {trading_pair}. Estimated Gas Cost: {gas_cost} ETH " + f" (gas limit: {gas_limit}, gas price: {gas_price})") tracked_order.update_exchange_order_id(hash) tracked_order.gas_price = gas_price if hash is not None: @@ -340,7 +381,7 @@ async def _update_order_status(self): for tracked_order in tracked_orders: order_id = await tracked_order.get_exchange_order_id() tasks.append(self._api_request("post", - "eth/get-receipt", + "eth/poll", {"txHash": order_id})) update_results = await safe_gather(*tasks, return_exceptions=True) for update_result in update_results: @@ -416,7 +457,7 @@ def has_allowances(self) -> bool: """ Checks if all tokens have allowance (an amount approved) """ - return len(self._allowances.values()) == len(self._token_addresses.values()) and \ + return len(self._allowances.values()) == len(self._tokens) and \ all(amount > s_decimal_0 for amount in self._allowances.values()) @property @@ -429,6 +470,7 @@ def status_dict(self) -> Dict[str, bool]: async def start_network(self): if self._trading_required: self._status_polling_task = safe_ensure_future(self._status_polling_loop()) + self._initiate_pool_task = safe_ensure_future(self.initiate_pool()) self._auto_approve_task = safe_ensure_future(self.auto_approve()) async def stop_network(self): @@ -438,6 +480,9 @@ async def stop_network(self): if self._auto_approve_task is not None: self._auto_approve_task.cancel() self._auto_approve_task = None + if self._initiate_pool_task is not None: + self._initiate_pool_task.cancel() + self._initiate_pool_task = None async def check_network(self) -> NetworkStatus: try: @@ -478,9 +523,6 @@ async def _status_polling_loop(self): app_warning_msg="Could not fetch balances from Gateway API.") await asyncio.sleep(0.5) - def get_token(self, token_address: str) -> str: - return [k for k, v in self._token_addresses.items() if v == token_address][0] - async def _update_balances(self, on_interval = False): """ Calls Eth API to update total and available balances. @@ -492,12 +534,10 @@ async def _update_balances(self, on_interval = False): local_asset_names = set(self._account_balances.keys()) remote_asset_names = set() resp_json = await self._api_request("post", - "eth/balances-2", - {"tokenAddressList": ("".join([tok + "," for tok in self._token_addresses.values()])).rstrip(","), - "tokenDecimalList": ("".join([str(dec) + "," for dec in self._token_decimals.values()])).rstrip(",")}) + "eth/balances", + {"tokenList": "[" + (",".join(['"' + t + '"' for t in self._tokens])) + "]"}) + for token, bal in resp_json["balances"].items(): - if len(token) > 4: - token = self.get_token(token) self._account_available_balances[token] = Decimal(str(bal)) self._account_balances[token] = Decimal(str(bal)) remote_asset_names.add(token) @@ -554,7 +594,7 @@ async def _api_request(self, err_msg = f" Message: {parsed_response['error']}" raise IOError(f"Error fetching data from {url}. HTTP status is {response.status}.{err_msg}") if "error" in parsed_response: - raise Exception(f"Error: {parsed_response['error']}") + raise Exception(f"Error: {parsed_response['error']} {parsed_response['message']}") return parsed_response diff --git a/hummingbot/connector/connector/terra/terra_connector.py b/hummingbot/connector/connector/terra/terra_connector.py index 1836750621..0f27b3e0ed 100644 --- a/hummingbot/connector/connector/terra/terra_connector.py +++ b/hummingbot/connector/connector/terra/terra_connector.py @@ -107,7 +107,7 @@ async def get_quote_price(self, trading_pair: str, is_buy: bool, amount: Decimal base, quote = trading_pair.split("-") side = "buy" if is_buy else "sell" - resp = await self._api_request("post", "terra/price", {"base": base, "quote": quote, "trade_type": side, + resp = await self._api_request("post", "terra/price", {"base": base, "quote": quote, "side": side, "amount": str(amount)}) txFee = resp["txFee"] / float(amount) price_with_txfee = resp["price"] + txFee if is_buy else resp["price"] - txFee @@ -185,9 +185,9 @@ async def _create_order(self, base, quote = trading_pair.split("-") api_params = {"base": base, "quote": quote, - "trade_type": "buy" if trade_type is TradeType.BUY else "sell", + "side": "buy" if trade_type is TradeType.BUY else "sell", "amount": str(amount), - "secret": self._terra_wallet_seeds, + "privateKey": self._terra_wallet_seeds, # "maxPrice": str(price), } self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount) diff --git a/hummingbot/connector/connector/uniswap/uniswap_connector.py b/hummingbot/connector/connector/uniswap/uniswap_connector.py index 2fef254945..c8277b65b1 100644 --- a/hummingbot/connector/connector/uniswap/uniswap_connector.py +++ b/hummingbot/connector/connector/uniswap/uniswap_connector.py @@ -7,7 +7,6 @@ import time import ssl import copy -import itertools as it from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL from hummingbot.core.utils import async_ttl_cache from hummingbot.core.network_iterator import NetworkStatus @@ -31,9 +30,9 @@ from hummingbot.connector.connector_base import ConnectorBase from hummingbot.connector.connector.uniswap.uniswap_in_flight_order import UniswapInFlightOrder from hummingbot.client.settings import GATEAWAY_CA_CERT_PATH, GATEAWAY_CLIENT_CERT_PATH, GATEAWAY_CLIENT_KEY_PATH -from hummingbot.core.utils.eth_gas_station_lookup import get_gas_price from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.client.config.config_helpers import get_erc20_token_addresses +from hummingbot.core.utils.ethereum import check_transaction_exceptions, fetch_trading_pairs +from hummingbot.client.config.fee_overrides_config_map import fee_overrides_config_map s_logger = None s_decimal_0 = Decimal("0") @@ -71,12 +70,9 @@ def __init__(self, """ super().__init__() self._trading_pairs = trading_pairs - tokens = set() + self._tokens = set() for trading_pair in trading_pairs: - tokens.update(set(trading_pair.split("-"))) - self._erc_20_token_list = self.token_list() - self._token_addresses = {t: l[0] for t, l in self._erc_20_token_list.items() if t in tokens} - self._token_decimals = {t: l[1] for t, l in self._erc_20_token_list.items() if t in tokens} + self._tokens.update(set(trading_pair.split("-"))) self._wallet_private_key = wallet_private_key self._ethereum_rpc_url = ethereum_rpc_url self._trading_required = trading_required @@ -84,10 +80,13 @@ def __init__(self, self._shared_client = None self._last_poll_timestamp = 0.0 self._last_balance_poll_timestamp = time.time() + self._last_est_gas_cost_reported = 0 self._in_flight_orders = {} self._allowances = {} self._status_polling_task = None self._auto_approve_task = None + self._initiate_pool_task = None + self._initiate_pool_status = None self._real_time_balance_update = False self._poll_notifier = None @@ -95,17 +94,9 @@ def __init__(self, def name(self): return "uniswap" - @staticmethod - def token_list(): - return get_erc20_token_addresses() - @staticmethod async def fetch_trading_pairs() -> List[str]: - token_list = UniswapConnector.token_list() - trading_pairs = [] - for base, quote in it.permutations(token_list.keys(), 2): - trading_pairs.append(f"{base}-{quote}") - return trading_pairs + return await fetch_trading_pairs() @property def limit_orders(self) -> List[LimitOrder]: @@ -114,6 +105,26 @@ def limit_orders(self) -> List[LimitOrder]: for in_flight_order in self._in_flight_orders.values() ] + async def initiate_pool(self) -> str: + """ + Initiate connector and start caching paths for trading_pairs + """ + try: + self.logger().info(f"Initializing Uniswap connector and paths for {self._trading_pairs} pairs.") + resp = await self._api_request("get", "eth/uniswap/start", + {"pairs": json.dumps(self._trading_pairs)}) + status = bool(str(resp["success"])) + if bool(str(resp["success"])): + self._initiate_pool_status = status + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().network( + f"Error initializing {self._trading_pairs} ", + exc_info=True, + app_warning_msg=str(e) + ) + async def auto_approve(self): """ Automatically approves Uniswap contract as a spender for token in trading pairs. @@ -137,9 +148,7 @@ async def approve_uniswap_spender(self, token_symbol: str) -> Decimal: """ resp = await self._api_request("post", "eth/approve", - {"tokenAddress": self._token_addresses[token_symbol], - "gasPrice": str(get_gas_price()), - "decimals": self._token_decimals[token_symbol], # if not supplied, gateway would treat it eth-like with 18 decimals + {"token": token_symbol, "connector": self.name}) amount_approved = Decimal(str(resp["amount"])) if amount_approved > 0: @@ -154,12 +163,11 @@ async def get_allowances(self) -> Dict[str, Decimal]: :return: A dictionary of token and its allowance (how much Uniswap can spend). """ ret_val = {} - resp = await self._api_request("post", "eth/allowances-2", - {"tokenAddressList": ("".join([tok + "," for tok in self._token_addresses.values()])).rstrip(","), - "tokenDecimalList": ("".join([str(dec) + "," for dec in self._token_decimals.values()])).rstrip(","), + resp = await self._api_request("post", "eth/allowances", + {"tokenList": "[" + (",".join(['"' + t + '"' for t in self._tokens])) + "]", "connector": self.name}) - for address, amount in resp["approvals"].items(): - ret_val[self.get_token(address)] = Decimal(str(amount)) + for token, amount in resp["approvals"].items(): + ret_val[token] = Decimal(str(amount)) return ret_val @async_ttl_cache(ttl=5, maxsize=10) @@ -176,12 +184,43 @@ async def get_quote_price(self, trading_pair: str, is_buy: bool, amount: Decimal base, quote = trading_pair.split("-") side = "buy" if is_buy else "sell" resp = await self._api_request("post", - f"uniswap/{side}-price", - {"base": self._token_addresses[base], - "quote": self._token_addresses[quote], + "eth/uniswap/price", + {"base": base, + "quote": quote, + "side": side.upper(), "amount": amount}) - if resp["price"] is not None: - return Decimal(str(resp["price"])) + required_items = ["price", "gasLimit", "gasPrice", "gasCost"] + if any(item not in resp.keys() for item in required_items): + if "info" in resp.keys(): + self.logger().info(f"Unable to get price. {resp['info']}") + else: + self.logger().info(f"Missing data from price result. Incomplete return result for ({resp.keys()})") + else: + gas_limit = resp["gasLimit"] + gas_price = resp["gasPrice"] + gas_cost = resp["gasCost"] + price = resp["price"] + account_standing = { + "allowances": self._allowances, + "balances": self._account_balances, + "base": base, + "quote": quote, + "amount": amount, + "side": side, + "gas_limit": gas_limit, + "gas_price": gas_price, + "gas_cost": gas_cost, + "price": price + } + exceptions = check_transaction_exceptions(account_standing) + for index in range(len(exceptions)): + self.logger().info(f"Warning! [{index+1}/{len(exceptions)}] {side} order - {exceptions[index]}") + + if price is not None and len(exceptions) == 0: + # TODO standardize quote price object to include price, fee, token, is fee part of quote. + fee_overrides_config_map["uniswap_maker_fee_amount"].value = Decimal(str(gas_cost)) + fee_overrides_config_map["uniswap_taker_fee_amount"].value = Decimal(str(gas_cost)) + return Decimal(str(price)) except asyncio.CancelledError: raise except Exception as e: @@ -251,21 +290,24 @@ async def _create_order(self, amount = self.quantize_order_amount(trading_pair, amount) price = self.quantize_order_price(trading_pair, price) base, quote = trading_pair.split("-") - gas_price = get_gas_price() - api_params = {"base": self._token_addresses[base], - "quote": self._token_addresses[quote], + api_params = {"base": base, + "quote": quote, + "side": trade_type.name.upper(), "amount": str(amount), - "maxPrice": str(price), - "gasPrice": str(gas_price), + "limitPrice": str(price), } - self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, gas_price) try: - order_result = await self._api_request("post", f"uniswap/{trade_type.name.lower()}", api_params) + order_result = await self._api_request("post", "eth/uniswap/trade", api_params) hash = order_result.get("txHash") + gas_price = order_result.get("gasPrice") + gas_limit = order_result.get("gasLimit") + gas_cost = order_result.get("gasCost") + self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, gas_price) tracked_order = self._in_flight_orders.get(order_id) if tracked_order is not None: self.logger().info(f"Created {trade_type.name} order {order_id} txHash: {hash} " - f"for {amount} {trading_pair}.") + f"for {amount} {trading_pair}. Estimated Gas Cost: {gas_cost} ETH " + f" (gas limit: {gas_limit}, gas price: {gas_price})") tracked_order.update_exchange_order_id(hash) tracked_order.gas_price = gas_price if hash is not None: @@ -333,7 +375,7 @@ async def _update_order_status(self): for tracked_order in tracked_orders: order_id = await tracked_order.get_exchange_order_id() tasks.append(self._api_request("post", - "eth/get-receipt", + "eth/poll", {"txHash": order_id})) update_results = await safe_gather(*tasks, return_exceptions=True) for update_result in update_results: @@ -409,7 +451,7 @@ def has_allowances(self) -> bool: """ Checks if all tokens have allowance (an amount approved) """ - return len(self._allowances.values()) == len(self._token_addresses.values()) and \ + return len(self._allowances.values()) == len(self._tokens) and \ all(amount > s_decimal_0 for amount in self._allowances.values()) @property @@ -422,6 +464,7 @@ def status_dict(self) -> Dict[str, bool]: async def start_network(self): if self._trading_required: self._status_polling_task = safe_ensure_future(self._status_polling_loop()) + self._initiate_pool_task = safe_ensure_future(self.initiate_pool()) self._auto_approve_task = safe_ensure_future(self.auto_approve()) async def stop_network(self): @@ -431,6 +474,9 @@ async def stop_network(self): if self._auto_approve_task is not None: self._auto_approve_task.cancel() self._auto_approve_task = None + if self._initiate_pool_task is not None: + self._initiate_pool_task.cancel() + self._initiate_pool_task = None async def check_network(self) -> NetworkStatus: try: @@ -458,7 +504,7 @@ async def _status_polling_loop(self): self._poll_notifier = asyncio.Event() await self._poll_notifier.wait() await safe_gather( - self._update_balances(), + self._update_balances(on_interval=True), self._update_order_status(), ) self._last_poll_timestamp = self.current_timestamp @@ -471,37 +517,32 @@ async def _status_polling_loop(self): app_warning_msg="Could not fetch balances from Gateway API.") await asyncio.sleep(0.5) - def get_token(self, token_address: str) -> str: - return [k for k, v in self._token_addresses.items() if v == token_address][0] - - async def _update_balances(self): + async def _update_balances(self, on_interval = False): """ Calls Eth API to update total and available balances. """ last_tick = self._last_balance_poll_timestamp current_tick = self.current_timestamp - if (current_tick - last_tick) > self.UPDATE_BALANCE_INTERVAL: + if not on_interval or (current_tick - last_tick) > self.UPDATE_BALANCE_INTERVAL: self._last_balance_poll_timestamp = current_tick - local_asset_names = set(self._account_balances.keys()) - remote_asset_names = set() - resp_json = await self._api_request("post", - "eth/balances-2", - {"tokenAddressList": ("".join([tok + "," for tok in self._token_addresses.values()])).rstrip(","), - "tokenDecimalList": ("".join([str(dec) + "," for dec in self._token_decimals.values()])).rstrip(",")}) - for token, bal in resp_json["balances"].items(): - if len(token) > 4: - token = self.get_token(token) - self._account_available_balances[token] = Decimal(str(bal)) - self._account_balances[token] = Decimal(str(bal)) - remote_asset_names.add(token) - - asset_names_to_remove = local_asset_names.difference(remote_asset_names) - for asset_name in asset_names_to_remove: - del self._account_available_balances[asset_name] - del self._account_balances[asset_name] - - self._in_flight_orders_snapshot = {k: copy.copy(v) for k, v in self._in_flight_orders.items()} - self._in_flight_orders_snapshot_timestamp = self.current_timestamp + local_asset_names = set(self._account_balances.keys()) + remote_asset_names = set() + resp_json = await self._api_request("post", + "eth/balances", + {"tokenList": "[" + (",".join(['"' + t + '"' for t in self._tokens])) + "]"}) + + for token, bal in resp_json["balances"].items(): + self._account_available_balances[token] = Decimal(str(bal)) + self._account_balances[token] = Decimal(str(bal)) + remote_asset_names.add(token) + + asset_names_to_remove = local_asset_names.difference(remote_asset_names) + for asset_name in asset_names_to_remove: + del self._account_available_balances[asset_name] + del self._account_balances[asset_name] + + self._in_flight_orders_snapshot = {k: copy.copy(v) for k, v in self._in_flight_orders.items()} + self._in_flight_orders_snapshot_timestamp = self.current_timestamp async def _http_client(self) -> aiohttp.ClientSession: """ @@ -547,7 +588,7 @@ async def _api_request(self, err_msg = f" Message: {parsed_response['error']}" raise IOError(f"Error fetching data from {url}. HTTP status is {response.status}.{err_msg}") if "error" in parsed_response: - raise Exception(f"Error: {parsed_response['error']}") + raise Exception(f"Error: {parsed_response['error']} {parsed_response['message']}") return parsed_response diff --git a/hummingbot/connector/connector_status.py b/hummingbot/connector/connector_status.py index 2f35999bb7..abe0f7dd2b 100644 --- a/hummingbot/connector/connector_status.py +++ b/hummingbot/connector/connector_status.py @@ -8,11 +8,12 @@ 'binance_perpetual_testnet': 'green', 'binance_us': 'yellow', 'bitfinex': 'yellow', - 'bitmax': 'green', + 'ascend_ex': 'green', 'bittrex': 'yellow', 'blocktane': 'green', 'celo': 'green', 'coinbase_pro': 'green', + 'coinzoom': 'yellow', 'crypto_com': 'yellow', 'dydx': 'green', 'eterbase': 'red', diff --git a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py index 94a4bbb2e5..252895e083 100644 --- a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py +++ b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py @@ -556,7 +556,7 @@ async def _user_stream_event_listener(self): update_data = event_message.get("a", {}) event_reason = update_data.get("m", {}) if event_reason == "FUNDING_FEE": - await self.get_funding_payment(event_message.get("E", int(time.time()))) + await self.get_funding_payment() else: # update balances for asset in update_data.get("B", []): @@ -933,7 +933,7 @@ async def get_funding_payment(self): funding_payment_tasks = [] for pair in self._trading_pairs: funding_payment_tasks.append(self.request(path="/fapi/v1/income", - params={"symbol": convert_to_exchange_trading_pair(pair), "incomeType": "FUNDING_FEE", "limit": 1}, + params={"symbol": convert_to_exchange_trading_pair(pair), "incomeType": "FUNDING_FEE", "limit": len(self._account_positions)}, method=MethodType.POST, add_timestamp=True, is_signed=True)) diff --git a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_order_book_tracker.py b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_order_book_tracker.py index e4df92f7b9..c25ba5444b 100644 --- a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_order_book_tracker.py +++ b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_order_book_tracker.py @@ -80,10 +80,8 @@ async def _order_book_diff_router(self): now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Diff messages process: %d, rejected: %d, queued: %d", - messages_accepted, - messages_rejected, - messages_queued) + self.logger().debug(f"Diff messages processed: {messages_accepted}, " + f"rejected: {messages_rejected}, queued: {messages_queued}") messages_accepted = 0 messages_rejected = 0 messages_queued = 0 @@ -128,14 +126,13 @@ async def _track_single_book(self, trading_pair: str): now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Processed %d order book diffs for %s.", - diff_messages_accepted, trading_pair) + self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") diff_messages_accepted = 0 last_message_timestamp = now elif message.type is OrderBookMessageType.SNAPSHOT: past_diffs: List[OrderBookMessage] = list(past_diffs_window) order_book.restore_from_snapshot_and_diffs(message, past_diffs) - self.logger().debug("Processed order book snapshot for %s.", trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise except Exception: diff --git a/hummingbot/connector/exchange/bitmax/__init__.py b/hummingbot/connector/exchange/ascend_ex/__init__.py similarity index 100% rename from hummingbot/connector/exchange/bitmax/__init__.py rename to hummingbot/connector/exchange/ascend_ex/__init__.py diff --git a/hummingbot/connector/exchange/bitmax/bitmax_active_order_tracker.pxd b/hummingbot/connector/exchange/ascend_ex/ascend_ex_active_order_tracker.pxd similarity index 90% rename from hummingbot/connector/exchange/bitmax/bitmax_active_order_tracker.pxd rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_active_order_tracker.pxd index fbc1eb3080..833d9864a0 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_active_order_tracker.pxd +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_active_order_tracker.pxd @@ -1,7 +1,7 @@ # distutils: language=c++ cimport numpy as np -cdef class BitmaxActiveOrderTracker: +cdef class AscendExActiveOrderTracker: cdef dict _active_bids cdef dict _active_asks diff --git a/hummingbot/connector/exchange/bitmax/bitmax_active_order_tracker.pyx b/hummingbot/connector/exchange/ascend_ex/ascend_ex_active_order_tracker.pyx similarity index 93% rename from hummingbot/connector/exchange/bitmax/bitmax_active_order_tracker.pyx rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_active_order_tracker.pyx index 092a97c45c..b80930b8b2 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_active_order_tracker.pyx +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_active_order_tracker.pyx @@ -12,12 +12,12 @@ from hummingbot.core.data_type.order_book_row import OrderBookRow _logger = None s_empty_diff = np.ndarray(shape=(0, 4), dtype="float64") -BitmaxOrderBookTrackingDictionary = Dict[Decimal, Dict[str, Dict[str, any]]] +AscendExOrderBookTrackingDictionary = Dict[Decimal, Dict[str, Dict[str, any]]] -cdef class BitmaxActiveOrderTracker: +cdef class AscendExActiveOrderTracker: def __init__(self, - active_asks: BitmaxOrderBookTrackingDictionary = None, - active_bids: BitmaxOrderBookTrackingDictionary = None): + active_asks: AscendExOrderBookTrackingDictionary = None, + active_bids: AscendExOrderBookTrackingDictionary = None): super().__init__() self._active_asks = active_asks or {} self._active_bids = active_bids or {} @@ -30,11 +30,11 @@ cdef class BitmaxActiveOrderTracker: return _logger @property - def active_asks(self) -> BitmaxOrderBookTrackingDictionary: + def active_asks(self) -> AscendExOrderBookTrackingDictionary: return self._active_asks @property - def active_bids(self) -> BitmaxOrderBookTrackingDictionary: + def active_bids(self) -> AscendExOrderBookTrackingDictionary: return self._active_bids # TODO: research this more diff --git a/hummingbot/connector/exchange/bitmax/bitmax_api_order_book_data_source.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_api_order_book_data_source.py similarity index 91% rename from hummingbot/connector/exchange/bitmax/bitmax_api_order_book_data_source.py rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_api_order_book_data_source.py index 306b0631c4..0c64a9c1c4 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_api_order_book_data_source.py @@ -13,13 +13,13 @@ from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource from hummingbot.core.utils.async_utils import safe_gather from hummingbot.logger import HummingbotLogger -from hummingbot.connector.exchange.bitmax.bitmax_active_order_tracker import BitmaxActiveOrderTracker -from hummingbot.connector.exchange.bitmax.bitmax_order_book import BitmaxOrderBook -from hummingbot.connector.exchange.bitmax.bitmax_utils import convert_from_exchange_trading_pair, convert_to_exchange_trading_pair -from hummingbot.connector.exchange.bitmax.bitmax_constants import EXCHANGE_NAME, REST_URL, WS_URL, PONG_PAYLOAD +from hummingbot.connector.exchange.ascend_ex.ascend_ex_active_order_tracker import AscendExActiveOrderTracker +from hummingbot.connector.exchange.ascend_ex.ascend_ex_order_book import AscendExOrderBook +from hummingbot.connector.exchange.ascend_ex.ascend_ex_utils import convert_from_exchange_trading_pair, convert_to_exchange_trading_pair +from hummingbot.connector.exchange.ascend_ex.ascend_ex_constants import EXCHANGE_NAME, REST_URL, WS_URL, PONG_PAYLOAD -class BitmaxAPIOrderBookDataSource(OrderBookTrackerDataSource): +class AscendExAPIOrderBookDataSource(OrderBookTrackerDataSource): MAX_RETRIES = 20 MESSAGE_TIMEOUT = 30.0 SNAPSHOT_TIMEOUT = 10.0 @@ -105,13 +105,13 @@ async def get_order_book_data(trading_pair: str) -> Dict[str, any]: async def get_new_order_book(self, trading_pair: str) -> OrderBook: snapshot: Dict[str, Any] = await self.get_order_book_data(trading_pair) snapshot_timestamp: float = snapshot.get("data").get("ts") - snapshot_msg: OrderBookMessage = BitmaxOrderBook.snapshot_message_from_exchange( + snapshot_msg: OrderBookMessage = AscendExOrderBook.snapshot_message_from_exchange( snapshot.get("data"), snapshot_timestamp, metadata={"trading_pair": trading_pair} ) order_book = self.order_book_create_function() - active_order_tracker: BitmaxActiveOrderTracker = BitmaxActiveOrderTracker() + active_order_tracker: AscendExActiveOrderTracker = AscendExActiveOrderTracker() bids, asks = active_order_tracker.convert_snapshot_message_to_order_book_row(snapshot_msg) order_book.apply_snapshot(bids, asks, snapshot_msg.update_id) return order_book @@ -141,7 +141,7 @@ async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asynci for trade in msg.get("data"): trade_timestamp: int = trade.get("ts") - trade_msg: OrderBookMessage = BitmaxOrderBook.trade_message_from_exchange( + trade_msg: OrderBookMessage = AscendExOrderBook.trade_message_from_exchange( trade, trade_timestamp, metadata={"trading_pair": trading_pair} @@ -181,7 +181,7 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp msg_timestamp: int = msg.get("data").get("ts") trading_pair: str = convert_from_exchange_trading_pair(msg.get("symbol")) - order_book_message: OrderBookMessage = BitmaxOrderBook.diff_message_from_exchange( + order_book_message: OrderBookMessage = AscendExOrderBook.diff_message_from_exchange( msg.get("data"), msg_timestamp, metadata={"trading_pair": trading_pair} @@ -207,7 +207,7 @@ async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, try: snapshot: Dict[str, any] = await self.get_order_book_data(trading_pair) snapshot_timestamp: float = snapshot.get("data").get("ts") - snapshot_msg: OrderBookMessage = BitmaxOrderBook.snapshot_message_from_exchange( + snapshot_msg: OrderBookMessage = AscendExOrderBook.snapshot_message_from_exchange( snapshot.get("data"), snapshot_timestamp, metadata={"trading_pair": trading_pair} diff --git a/hummingbot/connector/exchange/bitmax/bitmax_api_user_stream_data_source.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_api_user_stream_data_source.py similarity index 79% rename from hummingbot/connector/exchange/bitmax/bitmax_api_user_stream_data_source.py rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_api_user_stream_data_source.py index 10bbac6763..19571d90c8 100755 --- a/hummingbot/connector/exchange/bitmax/bitmax_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_api_user_stream_data_source.py @@ -9,11 +9,12 @@ from typing import Optional, List, AsyncIterable, Any from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource from hummingbot.logger import HummingbotLogger -from hummingbot.connector.exchange.bitmax.bitmax_auth import BitmaxAuth -from hummingbot.connector.exchange.bitmax.bitmax_constants import REST_URL, getWsUrlPriv, PONG_PAYLOAD +from hummingbot.connector.exchange.ascend_ex.ascend_ex_auth import AscendExAuth +from hummingbot.connector.exchange.ascend_ex.ascend_ex_constants import REST_URL, PONG_PAYLOAD +from hummingbot.connector.exchange.ascend_ex.ascend_ex_utils import get_ws_url_private -class BitmaxAPIUserStreamDataSource(UserStreamTrackerDataSource): +class AscendExAPIUserStreamDataSource(UserStreamTrackerDataSource): MAX_RETRIES = 20 MESSAGE_TIMEOUT = 10.0 PING_TIMEOUT = 5.0 @@ -26,8 +27,8 @@ def logger(cls) -> HummingbotLogger: cls._logger = logging.getLogger(__name__) return cls._logger - def __init__(self, bitmax_auth: BitmaxAuth, trading_pairs: Optional[List[str]] = []): - self._bitmax_auth: BitmaxAuth = bitmax_auth + def __init__(self, ascend_ex_auth: AscendExAuth, trading_pairs: Optional[List[str]] = []): + self._ascend_ex_auth: AscendExAuth = ascend_ex_auth self._trading_pairs = trading_pairs self._current_listen_key = None self._listen_for_user_stream_task = None @@ -50,18 +51,18 @@ async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: a while True: try: response = await aiohttp.ClientSession().get(f"{REST_URL}/info", headers={ - **self._bitmax_auth.get_headers(), - **self._bitmax_auth.get_auth_headers("info"), + **self._ascend_ex_auth.get_headers(), + **self._ascend_ex_auth.get_auth_headers("info"), }) info = await response.json() accountGroup = info.get("data").get("accountGroup") - headers = self._bitmax_auth.get_auth_headers("stream") + headers = self._ascend_ex_auth.get_auth_headers("stream") payload = { "op": "sub", "ch": "order:cash" } - async with websockets.connect(f"{getWsUrlPriv(accountGroup)}/stream", extra_headers=headers) as ws: + async with websockets.connect(f"{get_ws_url_private(accountGroup)}/stream", extra_headers=headers) as ws: try: ws: websockets.WebSocketClientProtocol = ws await ws.send(ujson.dumps(payload)) @@ -75,12 +76,12 @@ async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: a output.put_nowait(msg) except Exception: self.logger().error( - "Unexpected error when parsing Bitmax message. ", exc_info=True + "Unexpected error when parsing AscendEx message. ", exc_info=True ) raise except Exception: self.logger().error( - "Unexpected error while listening to Bitmax messages. ", exc_info=True + "Unexpected error while listening to AscendEx messages. ", exc_info=True ) raise finally: @@ -89,7 +90,7 @@ async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: a raise except Exception: self.logger().error( - "Unexpected error with Bitmax WebSocket connection. " "Retrying after 30 seconds...", exc_info=True + "Unexpected error with AscendEx WebSocket connection. " "Retrying after 30 seconds...", exc_info=True ) await asyncio.sleep(30.0) diff --git a/hummingbot/connector/exchange/bitmax/bitmax_auth.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_auth.py similarity index 79% rename from hummingbot/connector/exchange/bitmax/bitmax_auth.py rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_auth.py index 646fb13a8a..23b2c78f67 100755 --- a/hummingbot/connector/exchange/bitmax/bitmax_auth.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_auth.py @@ -1,13 +1,13 @@ import hmac import hashlib from typing import Dict, Any -from hummingbot.connector.exchange.bitmax.bitmax_utils import get_ms_timestamp +from hummingbot.connector.exchange.ascend_ex.ascend_ex_utils import get_ms_timestamp -class BitmaxAuth(): +class AscendExAuth(): """ - Auth class required by bitmax API - Learn more at https://bitmax-exchange.github.io/bitmax-pro-api/#authenticate-a-restful-request + Auth class required by AscendEx API + Learn more at https://ascendex.github.io/ascendex-pro-api/#authenticate-a-restful-request """ def __init__(self, api_key: str, secret_key: str): self.api_key = api_key @@ -39,7 +39,7 @@ def get_auth_headers( def get_headers(self) -> Dict[str, Any]: """ - Generates generic headers required by bitmax + Generates generic headers required by AscendEx :return: a dictionary of headers """ diff --git a/hummingbot/connector/exchange/ascend_ex/ascend_ex_constants.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_constants.py new file mode 100644 index 0000000000..27165b3257 --- /dev/null +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_constants.py @@ -0,0 +1,7 @@ +# A single source of truth for constant variables related to the exchange + + +EXCHANGE_NAME = "ascend_ex" +REST_URL = "https://ascendex.com/api/pro/v1" +WS_URL = "wss://ascendex.com/0/api/pro/v1/stream" +PONG_PAYLOAD = {"op": "pong"} diff --git a/hummingbot/connector/exchange/bitmax/bitmax_exchange.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_exchange.py similarity index 88% rename from hummingbot/connector/exchange/bitmax/bitmax_exchange.py rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_exchange.py index 4606157dd3..a8aac4e618 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_exchange.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_exchange.py @@ -36,25 +36,25 @@ TradeFee ) from hummingbot.connector.exchange_base import ExchangeBase -from hummingbot.connector.exchange.bitmax.bitmax_order_book_tracker import BitmaxOrderBookTracker -from hummingbot.connector.exchange.bitmax.bitmax_user_stream_tracker import BitmaxUserStreamTracker -from hummingbot.connector.exchange.bitmax.bitmax_auth import BitmaxAuth -from hummingbot.connector.exchange.bitmax.bitmax_in_flight_order import BitmaxInFlightOrder -from hummingbot.connector.exchange.bitmax import bitmax_utils -from hummingbot.connector.exchange.bitmax.bitmax_constants import EXCHANGE_NAME, REST_URL, getRestUrlPriv +from hummingbot.connector.exchange.ascend_ex.ascend_ex_order_book_tracker import AscendExOrderBookTracker +from hummingbot.connector.exchange.ascend_ex.ascend_ex_user_stream_tracker import AscendExUserStreamTracker +from hummingbot.connector.exchange.ascend_ex.ascend_ex_auth import AscendExAuth +from hummingbot.connector.exchange.ascend_ex.ascend_ex_in_flight_order import AscendExInFlightOrder +from hummingbot.connector.exchange.ascend_ex import ascend_ex_utils +from hummingbot.connector.exchange.ascend_ex.ascend_ex_constants import EXCHANGE_NAME, REST_URL from hummingbot.core.data_type.common import OpenOrder ctce_logger = None s_decimal_NaN = Decimal("nan") -BitmaxTradingRule = namedtuple("BitmaxTradingRule", "minNotional maxNotional") -BitmaxOrder = namedtuple("BitmaxOrder", "symbol price orderQty orderType avgPx cumFee cumFilledQty errorCode feeAsset lastExecTime orderId seqNum side status stopPrice execInst") -BitmaxBalance = namedtuple("BitmaxBalance", "asset availableBalance totalBalance") +AscendExTradingRule = namedtuple("AscendExTradingRule", "minNotional maxNotional") +AscendExOrder = namedtuple("AscendExOrder", "symbol price orderQty orderType avgPx cumFee cumFilledQty errorCode feeAsset lastExecTime orderId seqNum side status stopPrice execInst") +AscendExBalance = namedtuple("AscendExBalance", "asset availableBalance totalBalance") -class BitmaxExchange(ExchangeBase): +class AscendExExchange(ExchangeBase): """ - BitmaxExchange connects with Bitmax exchange and provides order book pricing, user account tracking and + AscendExExchange connects with AscendEx exchange and provides order book pricing, user account tracking and trading functionality. """ API_CALL_TIMEOUT = 10.0 @@ -70,31 +70,31 @@ def logger(cls) -> HummingbotLogger: return ctce_logger def __init__(self, - bitmax_api_key: str, - bitmax_secret_key: str, + ascend_ex_api_key: str, + ascend_ex_secret_key: str, trading_pairs: Optional[List[str]] = None, trading_required: bool = True ): """ - :param bitmax_api_key: The API key to connect to private Bitmax APIs. - :param bitmax_secret_key: The API secret. + :param ascend_ex_api_key: The API key to connect to private AscendEx APIs. + :param ascend_ex_secret_key: The API secret. :param trading_pairs: The market trading pairs which to track order book data. :param trading_required: Whether actual trading is needed. """ super().__init__() self._trading_required = trading_required self._trading_pairs = trading_pairs - self._bitmax_auth = BitmaxAuth(bitmax_api_key, bitmax_secret_key) - self._order_book_tracker = BitmaxOrderBookTracker(trading_pairs=trading_pairs) - self._user_stream_tracker = BitmaxUserStreamTracker(self._bitmax_auth, trading_pairs) + self._ascend_ex_auth = AscendExAuth(ascend_ex_api_key, ascend_ex_secret_key) + self._order_book_tracker = AscendExOrderBookTracker(trading_pairs=trading_pairs) + self._user_stream_tracker = AscendExUserStreamTracker(self._ascend_ex_auth, trading_pairs) self._ev_loop = asyncio.get_event_loop() self._shared_client = None self._poll_notifier = asyncio.Event() self._last_timestamp = 0 - self._in_flight_orders = {} # Dict[client_order_id:str, BitmaxInFlightOrder] + self._in_flight_orders = {} # Dict[client_order_id:str, AscendExInFlightOrder] self._order_not_found_records = {} # Dict[client_order_id:str, count:int] self._trading_rules = {} # Dict[trading_pair:str, TradingRule] - self._bitmax_trading_rules = {} # Dict[trading_pair:str, BitmaxTradingRule] + self._ascend_ex_trading_rules = {} # Dict[trading_pair:str, AscendExTradingRule] self._status_polling_task = None self._user_stream_event_listener_task = None self._trading_rules_polling_task = None @@ -115,7 +115,7 @@ def trading_rules(self) -> Dict[str, TradingRule]: return self._trading_rules @property - def in_flight_orders(self) -> Dict[str, BitmaxInFlightOrder]: + def in_flight_orders(self) -> Dict[str, AscendExInFlightOrder]: return self._in_flight_orders @property @@ -126,7 +126,7 @@ def status_dict(self) -> Dict[str, bool]: return { "order_books_initialized": self._order_book_tracker.ready, "account_balance": len(self._account_balances) > 0 if self._trading_required else True, - "trading_rule_initialized": len(self._trading_rules) > 0 and len(self._bitmax_trading_rules) > 0, + "trading_rule_initialized": len(self._trading_rules) > 0 and len(self._ascend_ex_trading_rules) > 0, "user_stream_initialized": self._user_stream_tracker.data_source.last_recv_time > 0 if self._trading_required else True, "account_data": self._account_group is not None and self._account_uid is not None @@ -165,7 +165,7 @@ def restore_tracking_states(self, saved_states: Dict[str, any]): :param saved_states: The saved tracking_states. """ self._in_flight_orders.update({ - key: BitmaxInFlightOrder.from_json(value) + key: AscendExInFlightOrder.from_json(value) for key, value in saved_states.items() }) @@ -258,22 +258,22 @@ async def _trading_rules_polling_loop(self): except Exception as e: self.logger().network(f"Unexpected error while fetching trading rules. Error: {str(e)}", exc_info=True, - app_warning_msg="Could not fetch new trading rules from Bitmax. " + app_warning_msg="Could not fetch new trading rules from AscendEx. " "Check network connection.") await asyncio.sleep(0.5) async def _update_trading_rules(self): instruments_info = await self._api_request("get", path_url="products") - [trading_rules, bitmax_trading_rules] = self._format_trading_rules(instruments_info) + [trading_rules, ascend_ex_trading_rules] = self._format_trading_rules(instruments_info) self._trading_rules.clear() self._trading_rules = trading_rules - self._bitmax_trading_rules.clear() - self._bitmax_trading_rules = bitmax_trading_rules + self._ascend_ex_trading_rules.clear() + self._ascend_ex_trading_rules = ascend_ex_trading_rules def _format_trading_rules( self, instruments_info: Dict[str, Any] - ) -> [Dict[str, TradingRule], Dict[str, Dict[str, BitmaxTradingRule]]]: + ) -> [Dict[str, TradingRule], Dict[str, Dict[str, AscendExTradingRule]]]: """ Converts json API response into a dictionary of trading rules. :param instruments_info: The json API response @@ -299,27 +299,27 @@ def _format_trading_rules( } """ trading_rules = {} - bitmax_trading_rules = {} + ascend_ex_trading_rules = {} for rule in instruments_info["data"]: try: - trading_pair = bitmax_utils.convert_from_exchange_trading_pair(rule["symbol"]) + trading_pair = ascend_ex_utils.convert_from_exchange_trading_pair(rule["symbol"]) trading_rules[trading_pair] = TradingRule( trading_pair, min_price_increment=Decimal(rule["tickSize"]), min_base_amount_increment=Decimal(rule["lotSize"]) ) - bitmax_trading_rules[trading_pair] = BitmaxTradingRule( + ascend_ex_trading_rules[trading_pair] = AscendExTradingRule( minNotional=Decimal(rule["minNotional"]), maxNotional=Decimal(rule["maxNotional"]) ) except Exception: self.logger().error(f"Error parsing the trading pair rule {rule}. Skipping.", exc_info=True) - return [trading_rules, bitmax_trading_rules] + return [trading_rules, ascend_ex_trading_rules] async def _update_account_data(self): headers = { - **self._bitmax_auth.get_headers(), - **self._bitmax_auth.get_auth_headers("info"), + **self._ascend_ex_auth.get_headers(), + **self._ascend_ex_auth.get_auth_headers("info"), } url = f"{REST_URL}/info" response = await aiohttp.ClientSession().get(url, headers=headers) @@ -359,16 +359,16 @@ async def _api_request(self, if (self._account_group) is None: await self._update_account_data() - url = f"{getRestUrlPriv(self._account_group)}/{path_url}" + url = f"{ascend_ex_utils.get_rest_url_private(self._account_group)}/{path_url}" headers = { - **self._bitmax_auth.get_headers(), - **self._bitmax_auth.get_auth_headers( + **self._ascend_ex_auth.get_headers(), + **self._ascend_ex_auth.get_auth_headers( path_url if force_auth_path_url is None else force_auth_path_url ), } else: url = f"{REST_URL}/{path_url}" - headers = self._bitmax_auth.get_headers() + headers = self._ascend_ex_auth.get_headers() client = await self._http_client() if method == "get": @@ -433,7 +433,7 @@ def buy(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, :param price: The price (note: this is no longer optional) :returns A new internal order id """ - client_order_id = bitmax_utils.gen_client_order_id(True, trading_pair) + client_order_id = ascend_ex_utils.gen_client_order_id(True, trading_pair) safe_ensure_future(self._create_order(TradeType.BUY, client_order_id, trading_pair, amount, order_type, price)) return client_order_id @@ -448,7 +448,7 @@ def sell(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, :param price: The price (note: this is no longer optional) :returns A new internal order id """ - client_order_id = bitmax_utils.gen_client_order_id(False, trading_pair) + client_order_id = ascend_ex_utils.gen_client_order_id(False, trading_pair) safe_ensure_future(self._create_order(TradeType.SELL, client_order_id, trading_pair, amount, order_type, price)) return client_order_id @@ -480,25 +480,25 @@ async def _create_order(self, """ if not order_type.is_limit_type(): raise Exception(f"Unsupported order type: {order_type}") - bitmax_trading_rule = self._bitmax_trading_rules[trading_pair] + ascend_ex_trading_rule = self._ascend_ex_trading_rules[trading_pair] amount = self.quantize_order_amount(trading_pair, amount) price = self.quantize_order_price(trading_pair, price) try: - # bitmax has a unique way of determening if the order has enough "worth" to be posted - # see https://bitmax-exchange.github.io/bitmax-pro-api/#place-order + # ascend_ex has a unique way of determening if the order has enough "worth" to be posted + # see https://ascendex.github.io/ascendex-pro-api/#place-order notional = Decimal(price * amount) - if notional < bitmax_trading_rule.minNotional or notional > bitmax_trading_rule.maxNotional: - raise ValueError(f"Notional amount {notional} is not withing the range of {bitmax_trading_rule.minNotional}-{bitmax_trading_rule.maxNotional}.") + if notional < ascend_ex_trading_rule.minNotional or notional > ascend_ex_trading_rule.maxNotional: + raise ValueError(f"Notional amount {notional} is not withing the range of {ascend_ex_trading_rule.minNotional}-{ascend_ex_trading_rule.maxNotional}.") # TODO: check balance - [exchange_order_id, timestamp] = bitmax_utils.gen_exchange_order_id(self._account_uid) + [exchange_order_id, timestamp] = ascend_ex_utils.gen_exchange_order_id(self._account_uid, order_id) api_params = { "id": exchange_order_id, "time": timestamp, - "symbol": bitmax_utils.convert_to_exchange_trading_pair(trading_pair), + "symbol": ascend_ex_utils.convert_to_exchange_trading_pair(trading_pair), "orderPrice": f"{price:f}", "orderQty": f"{amount:f}", "orderType": "limit", @@ -517,7 +517,6 @@ async def _create_order(self, await self._api_request("post", "cash/order", api_params, True, force_auth_path_url="order") tracked_order = self._in_flight_orders.get(order_id) - # tracked_order.update_exchange_order_id(exchange_order_id) if tracked_order is not None: self.logger().info(f"Created {order_type.name} {trade_type.name} order {order_id} for " @@ -540,7 +539,7 @@ async def _create_order(self, except Exception as e: self.stop_tracking_order(order_id) self.logger().network( - f"Error submitting {trade_type.name} {order_type.name} order to Bitmax for " + f"Error submitting {trade_type.name} {order_type.name} order to AscendEx for " f"{amount} {trading_pair} " f"{price}.", exc_info=True, @@ -560,7 +559,7 @@ def start_tracking_order(self, """ Starts tracking an order by simply adding it into _in_flight_orders dictionary. """ - self._in_flight_orders[order_id] = BitmaxInFlightOrder( + self._in_flight_orders[order_id] = AscendExInFlightOrder( client_order_id=order_id, exchange_order_id=exchange_order_id, trading_pair=trading_pair, @@ -596,9 +595,9 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: "delete", "cash/order", { - "symbol": bitmax_utils.convert_to_exchange_trading_pair(trading_pair), + "symbol": ascend_ex_utils.convert_to_exchange_trading_pair(trading_pair), "orderId": ex_order_id, - "time": bitmax_utils.get_ms_timestamp() + "time": ascend_ex_utils.get_ms_timestamp() }, True, force_auth_path_url="order" @@ -615,7 +614,7 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: self.logger().network( f"Failed to cancel order {order_id}: {str(e)}", exc_info=True, - app_warning_msg=f"Failed to cancel the order {order_id} on Bitmax. " + app_warning_msg=f"Failed to cancel the order {order_id} on AscendEx. " f"Check API key and network connection." ) @@ -639,7 +638,7 @@ async def _status_polling_loop(self): self.logger().error(str(e), exc_info=True) self.logger().network("Unexpected error while fetching account updates.", exc_info=True, - app_warning_msg="Could not fetch account updates from Bitmax. " + app_warning_msg="Could not fetch account updates from AscendEx. " "Check API key and network connection.") await asyncio.sleep(0.5) @@ -649,7 +648,7 @@ async def _update_balances(self): """ response = await self._api_request("get", "cash/balance", {}, True, force_auth_path_url="balance") balances = list(map( - lambda balance: BitmaxBalance( + lambda balance: AscendExBalance( balance["asset"], balance["availableBalance"], balance["totalBalance"] @@ -687,7 +686,7 @@ async def _update_order_status(self): continue order_data = response.get("data") - self._process_order_message(BitmaxOrder( + self._process_order_message(AscendExOrder( order_data["symbol"], order_data["price"], order_data["orderQty"], @@ -738,7 +737,7 @@ async def cancel_all(self, timeout_seconds: float): self.logger().network( "Failed to cancel all orders.", exc_info=True, - app_warning_msg="Failed to cancel all orders on Bitmax. Check API key and network connection." + app_warning_msg="Failed to cancel all orders on AscendEx. Check API key and network connection." ) return cancellation_results @@ -783,14 +782,14 @@ async def _iter_user_event_queue(self) -> AsyncIterable[Dict[str, any]]: self.logger().network( "Unknown error. Retrying after 1 seconds.", exc_info=True, - app_warning_msg="Could not fetch user events from Bitmax. Check API key and network connection." + app_warning_msg="Could not fetch user events from AscendEx. Check API key and network connection." ) await asyncio.sleep(1.0) async def _user_stream_event_listener(self): """ Listens to message in _user_stream_tracker.user_stream queue. The messages are put in by - BitmaxAPIUserStreamDataSource. + AscendExAPIUserStreamDataSource. """ async for event_message in self._iter_user_event_queue(): try: @@ -798,7 +797,7 @@ async def _user_stream_event_listener(self): order_data = event_message.get("data") trading_pair = order_data["s"] base_asset, quote_asset = tuple(asset for asset in trading_pair.split("/")) - self._process_order_message(BitmaxOrder( + self._process_order_message(AscendExOrder( trading_pair, order_data["p"], order_data["q"], @@ -817,12 +816,12 @@ async def _user_stream_event_listener(self): order_data["ei"] )) # Handles balance updates from orders. - base_asset_balance = BitmaxBalance( + base_asset_balance = AscendExBalance( base_asset, order_data["bab"], order_data["btb"] ) - quote_asset_balance = BitmaxBalance( + quote_asset_balance = AscendExBalance( quote_asset, order_data["qab"], order_data["qtb"] @@ -831,7 +830,7 @@ async def _user_stream_event_listener(self): elif event_message.get("m") == "balance": # Handles balance updates from Deposits/Withdrawals, Transfers between Cash and Margin Accounts balance_data = event_message.get("data") - balance = BitmaxBalance( + balance = AscendExBalance( balance_data["a"], balance_data["ab"], balance_data["tb"] @@ -869,7 +868,7 @@ async def get_open_orders(self) -> List[OpenOrder]: ret_val.append( OpenOrder( client_order_id=client_order_id, - trading_pair=bitmax_utils.convert_from_exchange_trading_pair(order["symbol"]), + trading_pair=ascend_ex_utils.convert_from_exchange_trading_pair(order["symbol"]), price=Decimal(str(order["price"])), amount=Decimal(str(order["orderQty"])), executed_amount=Decimal(str(order["cumFilledQty"])), @@ -882,7 +881,7 @@ async def get_open_orders(self) -> List[OpenOrder]: ) return ret_val - def _process_order_message(self, order_msg: BitmaxOrder): + def _process_order_message(self, order_msg: AscendExOrder): """ Updates in-flight order and triggers cancellation or failure event if needed. :param order_msg: The order response from either REST or web socket API (they are of the same format) @@ -917,7 +916,7 @@ def _process_order_message(self, order_msg: BitmaxOrder): self.stop_tracking_order(client_order_id) elif tracked_order.is_failure: self.logger().info(f"The market order {client_order_id} has failed according to order status API. " - f"Reason: {bitmax_utils.get_api_reason(order_msg.errorCode)}") + f"Reason: {ascend_ex_utils.get_api_reason(order_msg.errorCode)}") self.trigger_event(MarketEvent.OrderFailure, MarketOrderFailureEvent( self.current_timestamp, @@ -965,7 +964,7 @@ def _process_order_message(self, order_msg: BitmaxOrder): ) self.stop_tracking_order(client_order_id) - def _process_balances(self, balances: List[BitmaxBalance]): + def _process_balances(self, balances: List[AscendExBalance]): local_asset_names = set(self._account_balances.keys()) remote_asset_names = set() diff --git a/hummingbot/connector/exchange/bitmax/bitmax_in_flight_order.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_in_flight_order.py similarity index 95% rename from hummingbot/connector/exchange/bitmax/bitmax_in_flight_order.py rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_in_flight_order.py index b455433ba0..0654f2cba9 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_in_flight_order.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_in_flight_order.py @@ -12,7 +12,7 @@ from hummingbot.connector.in_flight_order_base import InFlightOrderBase -class BitmaxInFlightOrder(InFlightOrderBase): +class AscendExInFlightOrder(InFlightOrderBase): def __init__(self, client_order_id: str, exchange_order_id: Optional[str], @@ -53,7 +53,7 @@ def from_json(cls, data: Dict[str, Any]) -> InFlightOrderBase: :param data: json data from API :return: formatted InFlightOrder """ - retval = BitmaxInFlightOrder( + retval = AscendExInFlightOrder( data["client_order_id"], data["exchange_order_id"], data["trading_pair"], diff --git a/hummingbot/connector/exchange/bitmax/bitmax_order_book.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book.py similarity index 83% rename from hummingbot/connector/exchange/bitmax/bitmax_order_book.py rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book.py index be6702853d..88ebe8b0d1 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_order_book.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book.py @@ -1,24 +1,25 @@ #!/usr/bin/env python import logging -import hummingbot.connector.exchange.bitmax.bitmax_constants as constants +import hummingbot.connector.exchange.ascend_ex.ascend_ex_constants as constants from sqlalchemy.engine import RowProxy from typing import ( Optional, Dict, List, Any) -from hummingbot.logger import HummingbotLogger + from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.order_book_message import ( OrderBookMessage, OrderBookMessageType ) -from hummingbot.connector.exchange.bitmax.bitmax_order_book_message import BitmaxOrderBookMessage +from hummingbot.connector.exchange.ascend_ex.ascend_ex_order_book_message import AscendExOrderBookMessage +from hummingbot.logger import HummingbotLogger _logger = None -class BitmaxOrderBook(OrderBook): +class AscendExOrderBook(OrderBook): @classmethod def logger(cls) -> HummingbotLogger: global _logger @@ -35,13 +36,13 @@ def snapshot_message_from_exchange(cls, Convert json snapshot data into standard OrderBookMessage format :param msg: json snapshot data from live web socket stream :param timestamp: timestamp attached to incoming data - :return: BitmaxOrderBookMessage + :return: AscendExOrderBookMessage """ if metadata: msg.update(metadata) - return BitmaxOrderBookMessage( + return AscendExOrderBookMessage( message_type=OrderBookMessageType.SNAPSHOT, content=msg, timestamp=timestamp @@ -53,9 +54,9 @@ def snapshot_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = N *used for backtesting Convert a row of snapshot data into standard OrderBookMessage format :param record: a row of snapshot data from the database - :return: BitmaxOrderBookMessage + :return: AscendExOrderBookMessage """ - return BitmaxOrderBookMessage( + return AscendExOrderBookMessage( message_type=OrderBookMessageType.SNAPSHOT, content=record.json, timestamp=record.timestamp @@ -70,13 +71,13 @@ def diff_message_from_exchange(cls, Convert json diff data into standard OrderBookMessage format :param msg: json diff data from live web socket stream :param timestamp: timestamp attached to incoming data - :return: BitmaxOrderBookMessage + :return: AscendExOrderBookMessage """ if metadata: msg.update(metadata) - return BitmaxOrderBookMessage( + return AscendExOrderBookMessage( message_type=OrderBookMessageType.DIFF, content=msg, timestamp=timestamp @@ -88,9 +89,9 @@ def diff_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None) *used for backtesting Convert a row of diff data into standard OrderBookMessage format :param record: a row of diff data from the database - :return: BitmaxOrderBookMessage + :return: AscendExOrderBookMessage """ - return BitmaxOrderBookMessage( + return AscendExOrderBookMessage( message_type=OrderBookMessageType.DIFF, content=record.json, timestamp=record.timestamp @@ -104,7 +105,7 @@ def trade_message_from_exchange(cls, """ Convert a trade data into standard OrderBookMessage format :param record: a trade data from the database - :return: BitmaxOrderBookMessage + :return: AscendExOrderBookMessage """ if metadata: @@ -117,7 +118,7 @@ def trade_message_from_exchange(cls, "amount": msg.get("q"), }) - return BitmaxOrderBookMessage( + return AscendExOrderBookMessage( message_type=OrderBookMessageType.TRADE, content=msg, timestamp=timestamp @@ -129,9 +130,9 @@ def trade_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None *used for backtesting Convert a row of trade data into standard OrderBookMessage format :param record: a row of trade data from the database - :return: BitmaxOrderBookMessage + :return: AscendExOrderBookMessage """ - return BitmaxOrderBookMessage( + return AscendExOrderBookMessage( message_type=OrderBookMessageType.TRADE, content=record.json, timestamp=record.timestamp @@ -142,5 +143,5 @@ def from_snapshot(cls, snapshot: OrderBookMessage): raise NotImplementedError(constants.EXCHANGE_NAME + " order book needs to retain individual order data.") @classmethod - def restore_from_snapshot_and_diffs(self, snapshot: OrderBookMessage, diffs: List[OrderBookMessage]): + def restore_from_snapshot_and_diffs(cls, snapshot: OrderBookMessage, diffs: List[OrderBookMessage]): raise NotImplementedError(constants.EXCHANGE_NAME + " order book needs to retain individual order data.") diff --git a/hummingbot/connector/exchange/bitmax/bitmax_order_book_message.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book_message.py similarity index 95% rename from hummingbot/connector/exchange/bitmax/bitmax_order_book_message.py rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book_message.py index a00550b66b..c5b0ef2e13 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_order_book_message.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book_message.py @@ -13,7 +13,7 @@ ) -class BitmaxOrderBookMessage(OrderBookMessage): +class AscendExOrderBookMessage(OrderBookMessage): def __new__( cls, message_type: OrderBookMessageType, @@ -27,7 +27,7 @@ def __new__( raise ValueError("timestamp must not be None when initializing snapshot messages.") timestamp = content["timestamp"] - return super(BitmaxOrderBookMessage, cls).__new__( + return super(AscendExOrderBookMessage, cls).__new__( cls, message_type, content, timestamp=timestamp, *args, **kwargs ) diff --git a/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book_tracker.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book_tracker.py new file mode 100644 index 0000000000..e982f5c85b --- /dev/null +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book_tracker.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +import asyncio +import bisect +import logging +import time + +import hummingbot.connector.exchange.ascend_ex.ascend_ex_constants as constants + +from collections import defaultdict, deque +from typing import Optional, Dict, List, Deque + +from hummingbot.core.data_type.order_book_message import OrderBookMessageType +from hummingbot.core.data_type.order_book_tracker import OrderBookTracker +from hummingbot.connector.exchange.ascend_ex.ascend_ex_order_book_message import AscendExOrderBookMessage +from hummingbot.connector.exchange.ascend_ex.ascend_ex_active_order_tracker import AscendExActiveOrderTracker +from hummingbot.connector.exchange.ascend_ex.ascend_ex_api_order_book_data_source import AscendExAPIOrderBookDataSource +from hummingbot.connector.exchange.ascend_ex.ascend_ex_order_book import AscendExOrderBook +from hummingbot.logger import HummingbotLogger + + +class AscendExOrderBookTracker(OrderBookTracker): + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, trading_pairs: Optional[List[str]] = None,): + super().__init__(AscendExAPIOrderBookDataSource(trading_pairs), trading_pairs) + + self._ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + self._order_book_snapshot_stream: asyncio.Queue = asyncio.Queue() + self._order_book_diff_stream: asyncio.Queue = asyncio.Queue() + self._order_book_trade_stream: asyncio.Queue = asyncio.Queue() + self._process_msg_deque_task: Optional[asyncio.Task] = None + self._past_diffs_windows: Dict[str, Deque] = {} + self._order_books: Dict[str, AscendExOrderBook] = {} + self._saved_message_queues: Dict[str, Deque[AscendExOrderBookMessage]] = \ + defaultdict(lambda: deque(maxlen=1000)) + self._active_order_trackers: Dict[str, AscendExActiveOrderTracker] = defaultdict(AscendExActiveOrderTracker) + self._order_book_stream_listener_task: Optional[asyncio.Task] = None + self._order_book_trade_listener_task: Optional[asyncio.Task] = None + + @property + def exchange_name(self) -> str: + """ + Name of the current exchange + """ + return constants.EXCHANGE_NAME + + async def _track_single_book(self, trading_pair: str): + """ + Update an order book with changes from the latest batch of received messages + """ + past_diffs_window: Deque[AscendExOrderBookMessage] = deque() + self._past_diffs_windows[trading_pair] = past_diffs_window + + message_queue: asyncio.Queue = self._tracking_message_queues[trading_pair] + order_book: AscendExOrderBook = self._order_books[trading_pair] + active_order_tracker: AscendExActiveOrderTracker = self._active_order_trackers[trading_pair] + + last_message_timestamp: float = time.time() + diff_messages_accepted: int = 0 + + while True: + try: + message: AscendExOrderBookMessage = None + saved_messages: Deque[AscendExOrderBookMessage] = self._saved_message_queues[trading_pair] + # Process saved messages first if there are any + if len(saved_messages) > 0: + message = saved_messages.popleft() + else: + message = await message_queue.get() + + if message.type is OrderBookMessageType.DIFF: + bids, asks = active_order_tracker.convert_diff_message_to_order_book_row(message) + order_book.apply_diffs(bids, asks, message.update_id) + past_diffs_window.append(message) + while len(past_diffs_window) > self.PAST_DIFF_WINDOW_SIZE: + past_diffs_window.popleft() + diff_messages_accepted += 1 + + # Output some statistics periodically. + now: float = time.time() + if int(now / 60.0) > int(last_message_timestamp / 60.0): + self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") + diff_messages_accepted = 0 + last_message_timestamp = now + elif message.type is OrderBookMessageType.SNAPSHOT: + past_diffs: List[AscendExOrderBookMessage] = list(past_diffs_window) + # only replay diffs later than snapshot, first update active order with snapshot then replay diffs + replay_position = bisect.bisect_right(past_diffs, message) + replay_diffs = past_diffs[replay_position:] + s_bids, s_asks = active_order_tracker.convert_snapshot_message_to_order_book_row(message) + order_book.apply_snapshot(s_bids, s_asks, message.update_id) + for diff_message in replay_diffs: + d_bids, d_asks = active_order_tracker.convert_diff_message_to_order_book_row(diff_message) + order_book.apply_diffs(d_bids, d_asks, diff_message.update_id) + + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + f"Unexpected error processing order book messages for {trading_pair}.", + exc_info=True, + app_warning_msg="Unexpected error processing order book messages. Retrying after 5 seconds." + ) + await asyncio.sleep(5.0) diff --git a/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book_tracker_entry.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book_tracker_entry.py new file mode 100644 index 0000000000..ebefecfebb --- /dev/null +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book_tracker_entry.py @@ -0,0 +1,21 @@ +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_tracker_entry import OrderBookTrackerEntry +from hummingbot.connector.exchange.ascend_ex.ascend_ex_active_order_tracker import AscendExActiveOrderTracker + + +class AscendExOrderBookTrackerEntry(OrderBookTrackerEntry): + def __init__( + self, trading_pair: str, timestamp: float, order_book: OrderBook, active_order_tracker: AscendExActiveOrderTracker + ): + self._active_order_tracker = active_order_tracker + super(AscendExOrderBookTrackerEntry, self).__init__(trading_pair, timestamp, order_book) + + def __repr__(self) -> str: + return ( + f"AscendExOrderBookTrackerEntry(trading_pair='{self._trading_pair}', timestamp='{self._timestamp}', " + f"order_book='{self._order_book}')" + ) + + @property + def active_order_tracker(self) -> AscendExActiveOrderTracker: + return self._active_order_tracker diff --git a/hummingbot/connector/exchange/ascend_ex/ascend_ex_user_stream_tracker.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_user_stream_tracker.py new file mode 100644 index 0000000000..8557ca44db --- /dev/null +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_user_stream_tracker.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python + +import asyncio +import logging + +from typing import ( + Optional, + List, +) +from hummingbot.connector.exchange.ascend_ex.ascend_ex_api_user_stream_data_source import \ + AscendExAPIUserStreamDataSource +from hummingbot.connector.exchange.ascend_ex.ascend_ex_auth import AscendExAuth +from hummingbot.connector.exchange.ascend_ex.ascend_ex_constants import EXCHANGE_NAME +from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource +from hummingbot.core.data_type.user_stream_tracker import ( + UserStreamTracker +) +from hummingbot.core.utils.async_utils import ( + safe_ensure_future, + safe_gather, +) + +from hummingbot.logger import HummingbotLogger + + +class AscendExUserStreamTracker(UserStreamTracker): + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, + ascend_ex_auth: Optional[AscendExAuth] = None, + trading_pairs: Optional[List[str]] = []): + super().__init__() + self._ascend_ex_auth: AscendExAuth = ascend_ex_auth + self._trading_pairs: List[str] = trading_pairs + self._ev_loop: asyncio.events.AbstractEventLoop = asyncio.get_event_loop() + self._data_source: Optional[UserStreamTrackerDataSource] = None + self._user_stream_tracking_task: Optional[asyncio.Task] = None + + @property + def data_source(self) -> UserStreamTrackerDataSource: + """ + *required + Initializes a user stream data source (user specific order diffs from live socket stream) + :return: OrderBookTrackerDataSource + """ + if not self._data_source: + self._data_source = AscendExAPIUserStreamDataSource( + ascend_ex_auth=self._ascend_ex_auth, + trading_pairs=self._trading_pairs + ) + return self._data_source + + @property + def exchange_name(self) -> str: + """ + *required + Name of the current exchange + """ + return EXCHANGE_NAME + + async def start(self): + """ + *required + Start all listeners and tasks + """ + self._user_stream_tracking_task = safe_ensure_future( + self.data_source.listen_for_user_stream(self._ev_loop, self._user_stream) + ) + await safe_gather(self._user_stream_tracking_task) diff --git a/hummingbot/connector/exchange/bitmax/bitmax_utils.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_utils.py similarity index 57% rename from hummingbot/connector/exchange/bitmax/bitmax_utils.py rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_utils.py index 45f943688d..a8ac64400f 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_utils.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_utils.py @@ -15,7 +15,15 @@ DEFAULT_FEES = [0.1, 0.1] -HBOT_BROKER_ID = "hbot-" +HBOT_BROKER_ID = "HMBot" + + +def get_rest_url_private(account_id: int) -> str: + return f"https://ascendex.com/{account_id}/api/pro/v1" + + +def get_ws_url_private(account_id: int) -> str: + return f"wss://ascendex.com/{account_id}/api/pro/v1" def convert_from_exchange_trading_pair(exchange_trading_pair: str) -> str: @@ -35,29 +43,29 @@ def uuid32(): return ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(32)) -def derive_order_id(user_uid: str, cl_order_id: str, ts: int, order_src='a') -> str: +def derive_order_id(user_uid: str, cl_order_id: str, ts: int) -> str: """ Server order generator based on user info and input. :param user_uid: user uid :param cl_order_id: user random digital and number id :param ts: order timestamp in milliseconds - :param order_src: 'a' for rest api order, 's' for websocket order. :return: order id of length 32 """ - return (order_src + format(ts, 'x')[-11:] + user_uid[-11:] + cl_order_id[-9:])[:32] + return (HBOT_BROKER_ID + format(ts, 'x')[-11:] + user_uid[-11:] + cl_order_id[-5:])[:32] -def gen_exchange_order_id(userUid: str) -> Tuple[str, int]: +def gen_exchange_order_id(userUid: str, client_order_id: str) -> Tuple[str, int]: """ - Generate an order id - :param user_uid: user uid + Generates the exchange order id based on user uid and client order id. + :param user_uid: user uid, + :param client_order_id: client order id used for local order tracking :return: order id of length 32 """ time = get_ms_timestamp() return [ derive_order_id( userUid, - uuid32(), + client_order_id, time ), time @@ -66,20 +74,20 @@ def gen_exchange_order_id(userUid: str) -> Tuple[str, int]: def gen_client_order_id(is_buy: bool, trading_pair: str) -> str: side = "B" if is_buy else "S" - return f"{HBOT_BROKER_ID}{side}-{trading_pair}-{get_tracking_nonce()}" + return f"{HBOT_BROKER_ID}-{side}-{trading_pair}-{get_tracking_nonce()}" KEYS = { - "bitmax_api_key": - ConfigVar(key="bitmax_api_key", - prompt="Enter your Bitmax API key >>> ", - required_if=using_exchange("bitmax"), + "ascend_ex_api_key": + ConfigVar(key="ascend_ex_api_key", + prompt="Enter your AscendEx API key >>> ", + required_if=using_exchange("ascend_ex"), is_secure=True, is_connect_key=True), - "bitmax_secret_key": - ConfigVar(key="bitmax_secret_key", - prompt="Enter your Bitmax secret key >>> ", - required_if=using_exchange("bitmax"), + "ascend_ex_secret_key": + ConfigVar(key="ascend_ex_secret_key", + prompt="Enter your AscendEx secret key >>> ", + required_if=using_exchange("ascend_ex"), is_secure=True, is_connect_key=True), } diff --git a/hummingbot/connector/exchange/bamboo_relay/bamboo_relay_order_book_tracker.py b/hummingbot/connector/exchange/bamboo_relay/bamboo_relay_order_book_tracker.py index ef08e8c989..f29546a2b2 100644 --- a/hummingbot/connector/exchange/bamboo_relay/bamboo_relay_order_book_tracker.py +++ b/hummingbot/connector/exchange/bamboo_relay/bamboo_relay_order_book_tracker.py @@ -130,10 +130,8 @@ async def _order_book_diff_router(self): # Log some statistics. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Diff messages processed: %d, rejected: %d, queued: %d", - messages_accepted, - messages_rejected, - messages_queued) + self.logger().debug(f"Diff messages processed: {messages_accepted}, " + f"rejected: {messages_rejected}, queued: {messages_queued}") messages_accepted = 0 messages_rejected = 0 messages_queued = 0 @@ -175,7 +173,7 @@ async def _track_single_book(self, trading_pair: str): s_bids, s_asks = active_order_tracker.convert_snapshot_message_to_order_book_row(message) order_book.apply_snapshot(s_bids, s_asks, message.update_id) - self.logger().debug("Processed order book snapshot for %s.", trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise except Exception: diff --git a/hummingbot/connector/exchange/beaxy/beaxy_exchange.pyx b/hummingbot/connector/exchange/beaxy/beaxy_exchange.pyx index 53f4d09608..9fe3817fbf 100644 --- a/hummingbot/connector/exchange/beaxy/beaxy_exchange.pyx +++ b/hummingbot/connector/exchange/beaxy/beaxy_exchange.pyx @@ -363,8 +363,8 @@ cdef class BeaxyExchange(ExchangeBase): tracked_order.last_state = order_update['order_status'] if order_update['filled_size']: - execute_price = Decimal(order_update['limit_price'] if order_update['limit_price'] else order_update['average_price']) - execute_amount_diff = Decimal(order_update['filled_size']) - tracked_order.executed_amount_base + execute_price = Decimal(str(order_update['limit_price'] if order_update['limit_price'] else order_update['average_price'])) + execute_amount_diff = Decimal(str(order_update['filled_size'])) - tracked_order.executed_amount_base # Emit event if executed amount is greater than 0. if execute_amount_diff > s_decimal_0: @@ -398,9 +398,9 @@ cdef class BeaxyExchange(ExchangeBase): if tracked_order.is_done: if not tracked_order.is_failure and not tracked_order.is_cancelled: - new_confirmed_amount = Decimal(order_update['size']) + new_confirmed_amount = Decimal(str(order_update['size'])) execute_amount_diff = new_confirmed_amount - tracked_order.executed_amount_base - execute_price = Decimal(order_update['limit_price'] if order_update['limit_price'] else order_update['average_price']) + execute_price = Decimal(str(order_update['limit_price'] if order_update['limit_price'] else order_update['average_price'])) # Emit event if executed amount is greater than 0. if execute_amount_diff > s_decimal_0: @@ -748,8 +748,8 @@ cdef class BeaxyExchange(ExchangeBase): res = await self._api_request('get', BeaxyConstants.TradingApi.TRADE_SETTINGS_ENDPOINT) for symbol_data in res['symbols']: symbol = self.convert_from_exchange_trading_pair(symbol_data['name']) - self._maker_fee_percentage[symbol] = Decimal(symbol_data['maker_fee']) - self._taker_fee_percentage[symbol] = Decimal(symbol_data['taker_fee']) + self._maker_fee_percentage[symbol] = Decimal(str(symbol_data['maker_fee'])) + self._taker_fee_percentage[symbol] = Decimal(str(symbol_data['taker_fee'])) self._last_fee_percentage_update_timestamp = current_timestamp except asyncio.CancelledError: @@ -774,8 +774,8 @@ cdef class BeaxyExchange(ExchangeBase): for balance_entry in account_balances: asset_name = balance_entry['currency'] - available_balance = Decimal(balance_entry['available_balance']) - total_balance = Decimal(balance_entry['total_balance']) + available_balance = Decimal(str(balance_entry['available_balance'])) + total_balance = Decimal(str(balance_entry['total_balance'])) self._account_available_balances[asset_name] = available_balance self._account_balances[asset_name] = total_balance remote_asset_names.add(asset_name) @@ -855,8 +855,8 @@ cdef class BeaxyExchange(ExchangeBase): for msg in msgs: asset_name = msg['currency'] - available_balance = Decimal(msg['available_balance']) - total_balance = Decimal(msg['total_balance']) + available_balance = Decimal(str(msg['available_balance'])) + total_balance = Decimal(str(msg['total_balance'])) self._account_available_balances[asset_name] = available_balance self._account_balances[asset_name] = total_balance @@ -882,8 +882,8 @@ cdef class BeaxyExchange(ExchangeBase): execute_amount_diff = s_decimal_0 if order_status == 'partially_filled': - order_filled_size = Decimal(order['trade_size']) - execute_price = Decimal(order['trade_price']) + order_filled_size = Decimal(str(order['trade_size'])) + execute_price = Decimal(str(order['trade_price'])) execute_amount_diff = order_filled_size - tracked_order.executed_amount_base @@ -917,9 +917,9 @@ cdef class BeaxyExchange(ExchangeBase): elif order_status == 'completely_filled': - new_confirmed_amount = Decimal(order['size']) + new_confirmed_amount = Decimal(str(order['size'])) execute_amount_diff = new_confirmed_amount - tracked_order.executed_amount_base - execute_price = Decimal(order['limit_price'] if order['limit_price'] else order['average_price']) + execute_price = Decimal(str(order['limit_price'] if order['limit_price'] else order['average_price'])) # Emit event if executed amount is greater than 0. if execute_amount_diff > s_decimal_0: diff --git a/hummingbot/connector/exchange/beaxy/beaxy_order_book_tracker.py b/hummingbot/connector/exchange/beaxy/beaxy_order_book_tracker.py index d730dc97c7..3ba8c17580 100644 --- a/hummingbot/connector/exchange/beaxy/beaxy_order_book_tracker.py +++ b/hummingbot/connector/exchange/beaxy/beaxy_order_book_tracker.py @@ -82,10 +82,8 @@ async def _order_book_diff_router(self): # Log some statistics. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug('Messages processed: %d, rejected: %d, queued: %d', - messages_accepted, - messages_rejected, - messages_queued) + self.logger().debug(f"Messages processed: {messages_accepted}, " + f"rejected: {messages_rejected}, queued: {messages_queued}") messages_accepted = 0 messages_rejected = 0 messages_queued = 0 @@ -163,8 +161,7 @@ async def _track_single_book(self, trading_pair: str): # Output some statistics periodically. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug('Processed %d order book diffs for %s.', - diff_messages_accepted, trading_pair) + self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") diff_messages_accepted = 0 last_message_timestamp = now elif message.type is OrderBookMessageType.SNAPSHOT: @@ -178,7 +175,7 @@ async def _track_single_book(self, trading_pair: str): d_bids, d_asks = active_order_tracker.convert_diff_message_to_order_book_row(diff_message) order_book.apply_diffs(d_bids, d_asks, diff_message.update_id) - self.logger().debug('Processed order book snapshot for %s.', trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise except Exception: diff --git a/hummingbot/connector/exchange/binance/binance_exchange.pyx b/hummingbot/connector/exchange/binance/binance_exchange.pyx index ec048f8475..8601b0731d 100755 --- a/hummingbot/connector/exchange/binance/binance_exchange.pyx +++ b/hummingbot/connector/exchange/binance/binance_exchange.pyx @@ -402,7 +402,7 @@ cdef class BinanceExchange(ExchangeBase): trading_pairs = list(trading_pairs_to_order_map.keys()) tasks = [self.query_api(self._binance_client.get_my_trades, symbol=convert_to_exchange_trading_pair(trading_pair)) for trading_pair in trading_pairs] - self.logger().debug("Polling for order fills of %d trading pairs.", len(tasks)) + self.logger().debug(f"Polling for order fills of {len(tasks)} trading pairs.") results = await safe_gather(*tasks, return_exceptions=True) for trades, trading_pair in zip(results, trading_pairs): order_map = trading_pairs_to_order_map[trading_pair] @@ -448,7 +448,7 @@ cdef class BinanceExchange(ExchangeBase): trading_pairs = self._order_book_tracker._trading_pairs tasks = [self.query_api(self._binance_client.get_my_trades, symbol=convert_to_exchange_trading_pair(trading_pair)) for trading_pair in trading_pairs] - self.logger().debug("Polling for order fills of %d trading pairs.", len(tasks)) + self.logger().debug(f"Polling for order fills of {len(tasks)} trading pairs.") exchange_history = await safe_gather(*tasks, return_exceptions=True) for trades, trading_pair in zip(exchange_history, trading_pairs): if isinstance(trades, Exception): @@ -494,7 +494,7 @@ cdef class BinanceExchange(ExchangeBase): tasks = [self.query_api(self._binance_client.get_order, symbol=convert_to_exchange_trading_pair(o.trading_pair), origClientOrderId=o.client_order_id) for o in tracked_orders] - self.logger().debug("Polling for order status updates of %d orders.", len(tasks)) + self.logger().debug(f"Polling for order status updates of {len(tasks)} orders.") results = await safe_gather(*tasks, return_exceptions=True) for order_update, tracked_order in zip(results, tracked_orders): client_order_id = tracked_order.client_order_id diff --git a/hummingbot/connector/exchange/binance/binance_order_book_tracker.py b/hummingbot/connector/exchange/binance/binance_order_book_tracker.py index 40edaad388..55481ae47a 100644 --- a/hummingbot/connector/exchange/binance/binance_order_book_tracker.py +++ b/hummingbot/connector/exchange/binance/binance_order_book_tracker.py @@ -80,10 +80,8 @@ async def _order_book_diff_router(self): # Log some statistics. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Diff messages processed: %d, rejected: %d, queued: %d", - messages_accepted, - messages_rejected, - messages_queued) + self.logger().debug(f"Diff messages processed: {messages_accepted}, " + f"rejected: {messages_rejected}, queued: {messages_queued}") messages_accepted = 0 messages_rejected = 0 messages_queued = 0 @@ -129,14 +127,13 @@ async def _track_single_book(self, trading_pair: str): # Output some statistics periodically. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Processed %d order book diffs for %s.", - diff_messages_accepted, trading_pair) + self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") diff_messages_accepted = 0 last_message_timestamp = now elif message.type is OrderBookMessageType.SNAPSHOT: past_diffs: List[OrderBookMessage] = list(past_diffs_window) order_book.restore_from_snapshot_and_diffs(message, past_diffs) - self.logger().debug("Processed order book snapshot for %s.", trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise except Exception: diff --git a/hummingbot/connector/exchange/bitfinex/bitfinex_order_book_tracker.py b/hummingbot/connector/exchange/bitfinex/bitfinex_order_book_tracker.py index 9efdbb568a..68be932262 100644 --- a/hummingbot/connector/exchange/bitfinex/bitfinex_order_book_tracker.py +++ b/hummingbot/connector/exchange/bitfinex/bitfinex_order_book_tracker.py @@ -118,10 +118,8 @@ async def _order_book_diff_router(self): # Log some statistics. now: float = time.time() if int(now / CALC_STAT_MINUTE) > int(last_message_timestamp / CALC_STAT_MINUTE): - self.logger().debug("Diff messages processed: %d, rejected: %d, queued: %d", - messages_accepted, - messages_rejected, - messages_queued) + self.logger().debug(f"Diff messages processed: {messages_accepted}, " + f"rejected: {messages_rejected}, queued: {messages_queued}") messages_accepted = 0 messages_rejected = 0 messages_queued = 0 @@ -171,8 +169,7 @@ async def _track_single_book(self, trading_pair: str): # Output some statistics periodically. now: float = time.time() if int(now / CALC_STAT_MINUTE) > int(last_message_timestamp / CALC_STAT_MINUTE): - self.logger().debug( - "Processed %d order book diffs for %s.", diff_messages_accepted, trading_pair) + self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") diff_messages_accepted = 0 last_message_timestamp = now @@ -193,7 +190,7 @@ async def _track_single_book(self, trading_pair: str): ) order_book.apply_diffs(d_bids, d_asks, diff_message.update_id) - self.logger().debug("Processed order book snapshot for %s.", trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise except Exception as err: diff --git a/hummingbot/connector/exchange/bitmax/bitmax_constants.py b/hummingbot/connector/exchange/bitmax/bitmax_constants.py deleted file mode 100644 index 51ec061543..0000000000 --- a/hummingbot/connector/exchange/bitmax/bitmax_constants.py +++ /dev/null @@ -1,15 +0,0 @@ -# A single source of truth for constant variables related to the exchange - - -EXCHANGE_NAME = "bitmax" -REST_URL = "https://bitmax.io/api/pro/v1" -WS_URL = "wss://bitmax.io/1/api/pro/v1/stream" -PONG_PAYLOAD = {"op": "pong"} - - -def getRestUrlPriv(accountId: int) -> str: - return f"https://bitmax.io/{accountId}/api/pro/v1" - - -def getWsUrlPriv(accountId: int) -> str: - return f"wss://bitmax.io/{accountId}/api/pro/v1" diff --git a/hummingbot/connector/exchange/bitmax/bitmax_order_book_tracker_entry.py b/hummingbot/connector/exchange/bitmax/bitmax_order_book_tracker_entry.py deleted file mode 100644 index a97a33088a..0000000000 --- a/hummingbot/connector/exchange/bitmax/bitmax_order_book_tracker_entry.py +++ /dev/null @@ -1,21 +0,0 @@ -from hummingbot.core.data_type.order_book import OrderBook -from hummingbot.core.data_type.order_book_tracker_entry import OrderBookTrackerEntry -from hummingbot.connector.exchange.bitmax.bitmax_active_order_tracker import BitmaxActiveOrderTracker - - -class BitmaxOrderBookTrackerEntry(OrderBookTrackerEntry): - def __init__( - self, trading_pair: str, timestamp: float, order_book: OrderBook, active_order_tracker: BitmaxActiveOrderTracker - ): - self._active_order_tracker = active_order_tracker - super(BitmaxOrderBookTrackerEntry, self).__init__(trading_pair, timestamp, order_book) - - def __repr__(self) -> str: - return ( - f"BitmaxOrderBookTrackerEntry(trading_pair='{self._trading_pair}', timestamp='{self._timestamp}', " - f"order_book='{self._order_book}')" - ) - - @property - def active_order_tracker(self) -> BitmaxActiveOrderTracker: - return self._active_order_tracker diff --git a/hummingbot/connector/exchange/bittrex/bittrex_order_book_tracker.py b/hummingbot/connector/exchange/bittrex/bittrex_order_book_tracker.py index 4b779eba98..1a103368d3 100644 --- a/hummingbot/connector/exchange/bittrex/bittrex_order_book_tracker.py +++ b/hummingbot/connector/exchange/bittrex/bittrex_order_book_tracker.py @@ -184,7 +184,7 @@ async def _track_single_book(self, trading_pair: str): d_bids, d_asks = active_order_tracker.convert_diff_message_to_order_book_row(diff_message) order_book.apply_diffs(d_bids, d_asks, diff_message.update_id) - self.logger().debug("Processed order book snapshot for %s.", trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise except Exception: diff --git a/hummingbot/connector/exchange/blocktane/blocktane_order_book_tracker.py b/hummingbot/connector/exchange/blocktane/blocktane_order_book_tracker.py index beb3d3d66d..feec224339 100644 --- a/hummingbot/connector/exchange/blocktane/blocktane_order_book_tracker.py +++ b/hummingbot/connector/exchange/blocktane/blocktane_order_book_tracker.py @@ -77,10 +77,8 @@ async def _order_book_diff_router(self): # Log some statistics. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - # self.logger().debug("Diff messages processed: %d, rejected: %d, queued: %d", - # messages_accepted, - # messages_rejected, - # messages_queued) + # self.logger().debug(f"Diff messages processed: {messages_accepted}, " + # f"rejected: {messages_rejected}, queued: {messages_queued}") messages_accepted = 0 messages_rejected = 0 messages_queued = 0 @@ -127,14 +125,13 @@ async def _track_single_book(self, trading_pair: str): # Output some statistics periodically. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - # self.logger().debug("Processed %d order book diffs for %s.", - # diff_messages_accepted, trading_pair) + # self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") diff_messages_accepted = 0 last_message_timestamp = now elif message.type is OrderBookMessageType.SNAPSHOT: past_diffs: List[OrderBookMessage] = list(past_diffs_window) order_book.restore_from_snapshot_and_diffs(message, past_diffs) - # self.logger().debug("Processed order book snapshot for %s.", trading_pair) + # self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise except Exception: diff --git a/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_order_book_tracker.py b/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_order_book_tracker.py index ea8f5f4b6a..ebacd90075 100644 --- a/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_order_book_tracker.py +++ b/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_order_book_tracker.py @@ -99,10 +99,8 @@ async def _order_book_diff_router(self): # Log some statistics. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Diff messages processed: %d, rejected: %d, queued: %d", - messages_accepted, - messages_rejected, - messages_queued) + self.logger().debug(f"Diff messages processed: {messages_accepted}, " + f"rejected: {messages_rejected}, queued: {messages_queued}") messages_accepted = 0 messages_rejected = 0 messages_queued = 0 @@ -153,8 +151,7 @@ async def _track_single_book(self, trading_pair: str): # Output some statistics periodically. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Processed %d order book diffs for %s.", - diff_messages_accepted, trading_pair) + self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") diff_messages_accepted = 0 last_message_timestamp = now elif message.type is OrderBookMessageType.SNAPSHOT: @@ -168,7 +165,7 @@ async def _track_single_book(self, trading_pair: str): d_bids, d_asks = active_order_tracker.convert_diff_message_to_order_book_row(diff_message) order_book.apply_diffs(d_bids, d_asks, diff_message.update_id) - self.logger().debug("Processed order book snapshot for %s.", trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise except Exception: diff --git a/test/connector/exchange/bitmax/__init__.py b/hummingbot/connector/exchange/coinzoom/__init__.py similarity index 100% rename from test/connector/exchange/bitmax/__init__.py rename to hummingbot/connector/exchange/coinzoom/__init__.py diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd new file mode 100644 index 0000000000..881d7862df --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd @@ -0,0 +1,13 @@ +# distutils: language=c++ +cimport numpy as np + +cdef class CoinzoomActiveOrderTracker: + cdef dict _active_bids + cdef dict _active_asks + cdef dict _active_asks_ids + cdef dict _active_bids_ids + + cdef tuple c_convert_diff_message_to_np_arrays(self, object message) + cdef tuple c_convert_snapshot_message_to_np_arrays(self, object message) + # This method doesn't seem to be used anywhere at all + # cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx new file mode 100644 index 0000000000..a7e4fcb815 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx @@ -0,0 +1,157 @@ +# distutils: language=c++ +# distutils: sources=hummingbot/core/cpp/OrderBookEntry.cpp +import logging +import numpy as np +from decimal import Decimal +from typing import Dict +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.order_book_row import OrderBookRow + +_logger = None +s_empty_diff = np.ndarray(shape=(0, 4), dtype="float64") +CoinzoomOrderBookTrackingDictionary = Dict[Decimal, Dict[str, Dict[str, any]]] + +cdef class CoinzoomActiveOrderTracker: + def __init__(self, + active_asks: CoinzoomOrderBookTrackingDictionary = None, + active_bids: CoinzoomOrderBookTrackingDictionary = None): + super().__init__() + self._active_asks = active_asks or {} + self._active_bids = active_bids or {} + self._active_asks_ids = {} + self._active_bids_ids = {} + + @classmethod + def logger(cls) -> HummingbotLogger: + global _logger + if _logger is None: + _logger = logging.getLogger(__name__) + return _logger + + @property + def active_asks(self) -> CoinzoomOrderBookTrackingDictionary: + return self._active_asks + + @property + def active_bids(self) -> CoinzoomOrderBookTrackingDictionary: + return self._active_bids + + # TODO: research this more + def volume_for_ask_price(self, price) -> float: + return NotImplementedError + + # TODO: research this more + def volume_for_bid_price(self, price) -> float: + return NotImplementedError + + def get_rates_and_quantities(self, entry) -> tuple: + # price, quantity + return float(entry[0]), float(entry[1]) + + def get_rates_and_amts_with_ids(self, entry, id_list) -> tuple: + if len(entry) > 1: + price = float(entry[1]) + amount = float(entry[2]) + id_list[str(entry[0])] = price + else: + price = id_list.get(str(entry[0])) + amount = 0.0 + return price, amount + + cdef tuple c_convert_diff_message_to_np_arrays(self, object message): + cdef: + dict content = message.content + list content_keys = list(content.keys()) + list bid_entries = [] + list ask_entries = [] + str order_id + str order_side + str price_raw + object price + dict order_dict + double timestamp = message.timestamp + double amount = 0 + dict nps = {'bids': s_empty_diff, 'asks': s_empty_diff} + + if "b" in content_keys: + bid_entries = content["b"] + if "s" in content_keys: + ask_entries = content["s"] + + for entries, diff_key, id_list in [ + (bid_entries, 'bids', self._active_bids_ids), + (ask_entries, 'asks', self._active_asks_ids) + ]: + if len(entries) > 0: + nps[diff_key] = np.array( + [[timestamp, price, amount, message.update_id] + for price, amount in [self.get_rates_and_amts_with_ids(entry, id_list) for entry in entries] + if price is not None], + dtype="float64", ndmin=2 + ) + return nps['bids'], nps['asks'] + + cdef tuple c_convert_snapshot_message_to_np_arrays(self, object message): + cdef: + float price + float amount + str order_id + dict order_dict + + # Refresh all order tracking. + self._active_bids.clear() + self._active_asks.clear() + timestamp = message.timestamp + content = message.content + content_keys = list(content.keys()) + + if "bids" in content_keys: + for snapshot_orders, active_orders in [(content["bids"], self._active_bids), (content["asks"], self._active_asks)]: + for entry in snapshot_orders: + price, amount = self.get_rates_and_quantities(entry) + active_orders[price] = amount + else: + for snapshot_orders, active_orders, active_order_ids in [ + (content["b"], self._active_bids, self._active_bids_ids), + (content["s"], self._active_asks, self._active_asks_ids) + ]: + for entry in snapshot_orders: + price, amount = self.get_rates_and_amts_with_ids(entry, active_order_ids) + active_orders[price] = amount + + # Return the sorted snapshot tables. + cdef: + np.ndarray[np.float64_t, ndim=2] bids = np.array( + [[message.timestamp, float(price), float(self._active_bids[price]), message.update_id] + for price in sorted(self._active_bids.keys())], dtype='float64', ndmin=2) + np.ndarray[np.float64_t, ndim=2] asks = np.array( + [[message.timestamp, float(price), float(self._active_asks[price]), message.update_id] + for price in sorted(self._active_asks.keys(), reverse=True)], dtype='float64', ndmin=2) + + if bids.shape[1] != 4: + bids = bids.reshape((0, 4)) + if asks.shape[1] != 4: + asks = asks.reshape((0, 4)) + + return bids, asks + + # This method doesn't seem to be used anywhere at all + # cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message): + # cdef: + # double trade_type_value = 1.0 if message.content[4] == "BUY" else 2.0 + # list content = message.content + + # return np.array([message.timestamp, trade_type_value, float(content[1]), float(content[2])], + # dtype="float64") + + def convert_diff_message_to_order_book_row(self, message): + np_bids, np_asks = self.c_convert_diff_message_to_np_arrays(message) + bids_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_bids] + asks_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_asks] + return bids_row, asks_row + + def convert_snapshot_message_to_order_book_row(self, message): + np_bids, np_asks = self.c_convert_snapshot_message_to_np_arrays(message) + bids_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_bids] + asks_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_asks] + return bids_row, asks_row diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_api_order_book_data_source.py b/hummingbot/connector/exchange/coinzoom/coinzoom_api_order_book_data_source.py new file mode 100644 index 0000000000..49032d6f7f --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_api_order_book_data_source.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python +import asyncio +import logging +import time +import pandas as pd +from decimal import Decimal +from typing import Optional, List, Dict, Any +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_message import OrderBookMessage +from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource +from hummingbot.logger import HummingbotLogger +from .coinzoom_constants import Constants +from .coinzoom_active_order_tracker import CoinzoomActiveOrderTracker +from .coinzoom_order_book import CoinzoomOrderBook +from .coinzoom_websocket import CoinzoomWebsocket +from .coinzoom_utils import ( + convert_to_exchange_trading_pair, + convert_from_exchange_trading_pair, + api_call_with_retries, + CoinzoomAPIError, +) + + +class CoinzoomAPIOrderBookDataSource(OrderBookTrackerDataSource): + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, trading_pairs: List[str] = None): + super().__init__(trading_pairs) + self._trading_pairs: List[str] = trading_pairs + self._snapshot_msg: Dict[str, any] = {} + + @classmethod + async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, Decimal]: + results = {} + tickers: List[Dict[Any]] = await api_call_with_retries("GET", Constants.ENDPOINT["TICKER"]) + for trading_pair in trading_pairs: + ex_pair: str = convert_to_exchange_trading_pair(trading_pair, True) + ticker: Dict[Any] = list([tic for symbol, tic in tickers.items() if symbol == ex_pair])[0] + results[trading_pair]: Decimal = Decimal(str(ticker["last_price"])) + return results + + @staticmethod + async def fetch_trading_pairs() -> List[str]: + try: + symbols: List[Dict[str, Any]] = await api_call_with_retries("GET", Constants.ENDPOINT["SYMBOL"]) + trading_pairs: List[str] = list([convert_from_exchange_trading_pair(sym["symbol"]) for sym in symbols]) + # Filter out unmatched pairs so nothing breaks + return [sym for sym in trading_pairs if sym is not None] + except Exception: + # Do nothing if the request fails -- there will be no autocomplete for CoinZoom trading pairs + pass + return [] + + @staticmethod + async def get_order_book_data(trading_pair: str) -> Dict[str, any]: + """ + Get whole orderbook + """ + try: + ex_pair = convert_to_exchange_trading_pair(trading_pair, True) + ob_endpoint = Constants.ENDPOINT["ORDER_BOOK"].format(trading_pair=ex_pair) + orderbook_response: Dict[Any] = await api_call_with_retries("GET", ob_endpoint) + return orderbook_response + except CoinzoomAPIError as e: + err = e.error_payload.get('error', e.error_payload) + raise IOError( + f"Error fetching OrderBook for {trading_pair} at {Constants.EXCHANGE_NAME}. " + f"HTTP status is {e.error_payload['status']}. Error is {err.get('message', str(err))}.") + + async def get_new_order_book(self, trading_pair: str) -> OrderBook: + snapshot: Dict[str, Any] = await self.get_order_book_data(trading_pair) + snapshot_timestamp: float = float(snapshot['timestamp']) + snapshot_msg: OrderBookMessage = CoinzoomOrderBook.snapshot_message_from_exchange( + snapshot, + snapshot_timestamp, + metadata={"trading_pair": trading_pair}) + order_book = self.order_book_create_function() + active_order_tracker: CoinzoomActiveOrderTracker = CoinzoomActiveOrderTracker() + bids, asks = active_order_tracker.convert_snapshot_message_to_order_book_row(snapshot_msg) + order_book.apply_snapshot(bids, asks, snapshot_msg.update_id) + return order_book + + async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for trades using websocket trade channel + """ + while True: + try: + ws = CoinzoomWebsocket() + await ws.connect() + + for pair in self._trading_pairs: + await ws.subscribe({Constants.WS_SUB["TRADES"]: {'symbol': convert_to_exchange_trading_pair(pair)}}) + + async for response in ws.on_message(): + msg_keys = list(response.keys()) if response is not None else [] + + if not Constants.WS_METHODS["TRADES_UPDATE"] in msg_keys: + continue + + trade: List[Any] = response[Constants.WS_METHODS["TRADES_UPDATE"]] + trade_msg: OrderBookMessage = CoinzoomOrderBook.trade_message_from_exchange(trade) + output.put_nowait(trade_msg) + + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error.", exc_info=True) + raise + await asyncio.sleep(5.0) + finally: + await ws.disconnect() + + async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for orderbook diffs using websocket book channel + """ + while True: + try: + ws = CoinzoomWebsocket() + await ws.connect() + + order_book_methods = [ + Constants.WS_METHODS['ORDERS_SNAPSHOT'], + Constants.WS_METHODS['ORDERS_UPDATE'], + ] + + for pair in self._trading_pairs: + ex_pair = convert_to_exchange_trading_pair(pair) + ws_stream = { + Constants.WS_SUB["ORDERS"]: { + 'requestId': ex_pair, + 'symbol': ex_pair, + 'aggregate': False, + 'depth': 0, + } + } + await ws.subscribe(ws_stream) + + async for response in ws.on_message(): + msg_keys = list(response.keys()) if response is not None else [] + + method_key = [key for key in msg_keys if key in order_book_methods] + + if len(method_key) != 1: + continue + + method: str = method_key[0] + order_book_data: dict = response + timestamp: int = int(time.time() * 1e3) + pair: str = convert_from_exchange_trading_pair(response[method]) + + order_book_msg_cls = (CoinzoomOrderBook.diff_message_from_exchange + if method == Constants.WS_METHODS['ORDERS_UPDATE'] else + CoinzoomOrderBook.snapshot_message_from_exchange) + + orderbook_msg: OrderBookMessage = order_book_msg_cls( + order_book_data, + timestamp, + metadata={"trading_pair": pair}) + output.put_nowait(orderbook_msg) + + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unexpected error with WebSocket connection.", exc_info=True, + app_warning_msg="Unexpected error with WebSocket connection. Retrying in 30 seconds. " + "Check network connection.") + await asyncio.sleep(30.0) + finally: + await ws.disconnect() + + async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for orderbook snapshots by fetching orderbook + """ + while True: + try: + for trading_pair in self._trading_pairs: + try: + snapshot: Dict[str, any] = await self.get_order_book_data(trading_pair) + snapshot_msg: OrderBookMessage = CoinzoomOrderBook.snapshot_message_from_exchange( + snapshot, + snapshot['timestamp'], + metadata={"trading_pair": trading_pair} + ) + output.put_nowait(snapshot_msg) + self.logger().debug(f"Saved order book snapshot for {trading_pair}") + # Be careful not to go above API rate limits. + await asyncio.sleep(5.0) + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unexpected error with WebSocket connection.", exc_info=True, + app_warning_msg="Unexpected error with WebSocket connection. Retrying in 5 seconds. " + "Check network connection.") + await asyncio.sleep(5.0) + this_hour: pd.Timestamp = pd.Timestamp.utcnow().replace(minute=0, second=0, microsecond=0) + next_hour: pd.Timestamp = this_hour + pd.Timedelta(hours=1) + delta: float = next_hour.timestamp() - time.time() + await asyncio.sleep(delta) + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error.", exc_info=True) + await asyncio.sleep(5.0) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py b/hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py new file mode 100755 index 0000000000..7aede77e89 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +import time +import asyncio +import logging +from typing import ( + Any, + AsyncIterable, + List, + Optional, +) +from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource +from hummingbot.logger import HummingbotLogger +from .coinzoom_constants import Constants +from .coinzoom_auth import CoinzoomAuth +from .coinzoom_utils import CoinzoomAPIError +from .coinzoom_websocket import CoinzoomWebsocket + + +class CoinzoomAPIUserStreamDataSource(UserStreamTrackerDataSource): + + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, coinzoom_auth: CoinzoomAuth, trading_pairs: Optional[List[str]] = []): + self._coinzoom_auth: CoinzoomAuth = coinzoom_auth + self._ws: CoinzoomWebsocket = None + self._trading_pairs = trading_pairs + self._current_listen_key = None + self._listen_for_user_stream_task = None + self._last_recv_time: float = 0 + super().__init__() + + @property + def last_recv_time(self) -> float: + return self._last_recv_time + + async def _ws_request_balances(self): + return await self._ws.request(Constants.WS_METHODS["USER_BALANCE"]) + + async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: + """ + Subscribe to active orders via web socket + """ + + try: + self._ws = CoinzoomWebsocket(self._coinzoom_auth) + + await self._ws.connect() + + await self._ws.subscribe({Constants.WS_SUB["USER_ORDERS_TRADES"]: {}}) + + event_methods = [ + Constants.WS_METHODS["USER_ORDERS"], + # We don't need to know about pending cancels + # Constants.WS_METHODS["USER_ORDERS_CANCEL"], + ] + + async for msg in self._ws.on_message(): + self._last_recv_time = time.time() + + msg_keys = list(msg.keys()) if msg is not None else [] + + if not any(ws_method in msg_keys for ws_method in event_methods): + continue + yield msg + except Exception as e: + raise e + finally: + await self._ws.disconnect() + await asyncio.sleep(5) + + async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue) -> AsyncIterable[Any]: + """ + *required + Subscribe to user stream via web socket, and keep the connection open for incoming messages + :param ev_loop: ev_loop to execute this function in + :param output: an async queue where the incoming messages are stored + """ + + while True: + try: + async for msg in self._listen_to_orders_trades_balances(): + output.put_nowait(msg) + except asyncio.CancelledError: + raise + except CoinzoomAPIError as e: + self.logger().error(e.error_payload.get('error'), exc_info=True) + raise + except Exception: + self.logger().error( + f"Unexpected error with {Constants.EXCHANGE_NAME} WebSocket connection. " + "Retrying after 30 seconds...", exc_info=True) + await asyncio.sleep(30.0) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_auth.py b/hummingbot/connector/exchange/coinzoom/coinzoom_auth.py new file mode 100755 index 0000000000..9379f3716b --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_auth.py @@ -0,0 +1,31 @@ +from typing import Dict, Any + + +class CoinzoomAuth(): + """ + Auth class required by CoinZoom API + Learn more at https://exchange-docs.crypto.com/#digital-signature + """ + def __init__(self, api_key: str, secret_key: str, username: str): + self.api_key = api_key + self.secret_key = secret_key + self.username = username + + def get_ws_params(self) -> Dict[str, str]: + return { + "apiKey": str(self.api_key), + "secretKey": str(self.secret_key), + } + + def get_headers(self) -> Dict[str, Any]: + """ + Generates authentication headers required by CoinZoom + :return: a dictionary of auth headers + """ + headers = { + "Content-Type": "application/json", + "Coinzoom-Api-Key": str(self.api_key), + "Coinzoom-Api-Secret": str(self.secret_key), + "User-Agent": f"hummingbot ZoomMe: {self.username}" + } + return headers diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py new file mode 100644 index 0000000000..0cad1cb049 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py @@ -0,0 +1,60 @@ +# A single source of truth for constant variables related to the exchange +class Constants: + """ + API Documentation Links: + https://api-docs.coinzoom.com/ + https://api-markets.coinzoom.com/ + """ + EXCHANGE_NAME = "coinzoom" + REST_URL = "https://api.coinzoom.com/api/v1/public" + # REST_URL = "https://api.stage.coinzoom.com/api/v1/public" + WS_PRIVATE_URL = "wss://api.coinzoom.com/api/v1/public/market/data/stream" + # WS_PRIVATE_URL = "wss://api.stage.coinzoom.com/api/v1/public/market/data/stream" + WS_PUBLIC_URL = "wss://api.coinzoom.com/api/v1/public/market/data/stream" + # WS_PUBLIC_URL = "wss://api.stage.coinzoom.com/api/v1/public/market/data/stream" + + HBOT_BROKER_ID = "CZ_API_HBOT" + + ENDPOINT = { + # Public Endpoints + "TICKER": "marketwatch/ticker", + "SYMBOL": "instruments", + "ORDER_BOOK": "marketwatch/orderbook/{trading_pair}/150/2", + "ORDER_CREATE": "orders/new", + "ORDER_DELETE": "orders/cancel", + "ORDER_STATUS": "orders/list", + "USER_ORDERS": "orders/list", + "USER_BALANCES": "ledger/list", + } + + WS_SUB = { + "TRADES": "TradeSummaryRequest", + "ORDERS": "OrderBookRequest", + "USER_ORDERS_TRADES": "OrderUpdateRequest", + + } + + WS_METHODS = { + "ORDERS_SNAPSHOT": "ob", + "ORDERS_UPDATE": "oi", + "TRADES_UPDATE": "ts", + "USER_BALANCE": "getTradingBalance", + "USER_ORDERS": "OrderResponse", + "USER_ORDERS_CANCEL": "OrderCancelResponse", + } + + # Timeouts + MESSAGE_TIMEOUT = 30.0 + PING_TIMEOUT = 10.0 + API_CALL_TIMEOUT = 10.0 + API_MAX_RETRIES = 4 + + # Intervals + # Only used when nothing is received from WS + SHORT_POLL_INTERVAL = 5.0 + # One minute should be fine since we request balance updates on order updates + LONG_POLL_INTERVAL = 60.0 + # One minute should be fine for order status since we get these via WS + UPDATE_ORDER_STATUS_INTERVAL = 60.0 + # 10 minute interval to update trading rules, these would likely never change whilst running. + INTERVAL_TRADING_RULES = 600 diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py new file mode 100644 index 0000000000..65108d7475 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py @@ -0,0 +1,919 @@ +import logging +from typing import ( + Dict, + List, + Optional, + Any, + AsyncIterable, +) +from decimal import Decimal +import asyncio +import aiohttp +import math +import time +import ujson +from async_timeout import timeout + +from hummingbot.core.network_iterator import NetworkStatus +from hummingbot.logger import HummingbotLogger +from hummingbot.core.clock import Clock +from hummingbot.core.utils.asyncio_throttle import Throttler +from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather +from hummingbot.connector.trading_rule import TradingRule +from hummingbot.core.data_type.cancellation_result import CancellationResult +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.limit_order import LimitOrder +from hummingbot.core.event.events import ( + MarketEvent, + BuyOrderCompletedEvent, + SellOrderCompletedEvent, + OrderFilledEvent, + OrderCancelledEvent, + BuyOrderCreatedEvent, + SellOrderCreatedEvent, + MarketOrderFailureEvent, + OrderType, + TradeType, + TradeFee +) +from hummingbot.connector.exchange_base import ExchangeBase +from hummingbot.connector.exchange.coinzoom.coinzoom_order_book_tracker import CoinzoomOrderBookTracker +from hummingbot.connector.exchange.coinzoom.coinzoom_user_stream_tracker import CoinzoomUserStreamTracker +from hummingbot.connector.exchange.coinzoom.coinzoom_auth import CoinzoomAuth +from hummingbot.connector.exchange.coinzoom.coinzoom_in_flight_order import CoinzoomInFlightOrder +from hummingbot.connector.exchange.coinzoom.coinzoom_utils import ( + convert_from_exchange_trading_pair, + convert_to_exchange_trading_pair, + get_new_client_order_id, + aiohttp_response_with_errors, + retry_sleep_time, + str_date_to_ts, + CoinzoomAPIError, +) +from hummingbot.connector.exchange.coinzoom.coinzoom_constants import Constants +from hummingbot.core.data_type.common import OpenOrder +ctce_logger = None +s_decimal_NaN = Decimal("nan") + + +class CoinzoomExchange(ExchangeBase): + """ + CoinzoomExchange connects with CoinZoom exchange and provides order book pricing, user account tracking and + trading functionality. + """ + ORDER_NOT_EXIST_CONFIRMATION_COUNT = 3 + ORDER_NOT_EXIST_CANCEL_COUNT = 2 + + @classmethod + def logger(cls) -> HummingbotLogger: + global ctce_logger + if ctce_logger is None: + ctce_logger = logging.getLogger(__name__) + return ctce_logger + + def __init__(self, + coinzoom_api_key: str, + coinzoom_secret_key: str, + coinzoom_username: str, + trading_pairs: Optional[List[str]] = None, + trading_required: bool = True + ): + """ + :param coinzoom_api_key: The API key to connect to private CoinZoom APIs. + :param coinzoom_secret_key: The API secret. + :param coinzoom_username: The ZoomMe Username. + :param trading_pairs: The market trading pairs which to track order book data. + :param trading_required: Whether actual trading is needed. + """ + super().__init__() + self._trading_required = trading_required + self._trading_pairs = trading_pairs + self._coinzoom_auth = CoinzoomAuth(coinzoom_api_key, coinzoom_secret_key, coinzoom_username) + self._order_book_tracker = CoinzoomOrderBookTracker(trading_pairs=trading_pairs) + self._user_stream_tracker = CoinzoomUserStreamTracker(self._coinzoom_auth, trading_pairs) + self._ev_loop = asyncio.get_event_loop() + self._shared_client = None + self._poll_notifier = asyncio.Event() + self._last_timestamp = 0 + self._in_flight_orders = {} # Dict[client_order_id:str, CoinzoomInFlightOrder] + self._order_not_found_records = {} # Dict[client_order_id:str, count:int] + self._trading_rules = {} # Dict[trading_pair:str, TradingRule] + self._status_polling_task = None + self._user_stream_event_listener_task = None + self._trading_rules_polling_task = None + self._last_poll_timestamp = 0 + self._throttler = Throttler(rate_limit = (8.0, 6)) + + @property + def name(self) -> str: + return "coinzoom" + + @property + def order_books(self) -> Dict[str, OrderBook]: + return self._order_book_tracker.order_books + + @property + def trading_rules(self) -> Dict[str, TradingRule]: + return self._trading_rules + + @property + def in_flight_orders(self) -> Dict[str, CoinzoomInFlightOrder]: + return self._in_flight_orders + + @property + def status_dict(self) -> Dict[str, bool]: + """ + A dictionary of statuses of various connector's components. + """ + return { + "order_books_initialized": self._order_book_tracker.ready, + "account_balance": len(self._account_balances) > 0 if self._trading_required else True, + "trading_rule_initialized": len(self._trading_rules) > 0, + "user_stream_initialized": + self._user_stream_tracker.data_source.last_recv_time > 0 if self._trading_required else True, + } + + @property + def ready(self) -> bool: + """ + :return True when all statuses pass, this might take 5-10 seconds for all the connector's components and + services to be ready. + """ + return all(self.status_dict.values()) + + @property + def limit_orders(self) -> List[LimitOrder]: + return [ + in_flight_order.to_limit_order() + for in_flight_order in self._in_flight_orders.values() + ] + + @property + def tracking_states(self) -> Dict[str, any]: + """ + :return active in-flight orders in json format, is used to save in sqlite db. + """ + return { + key: value.to_json() + for key, value in self._in_flight_orders.items() + if not value.is_done + } + + def restore_tracking_states(self, saved_states: Dict[str, any]): + """ + Restore in-flight orders from saved tracking states, this is st the connector can pick up on where it left off + when it disconnects. + :param saved_states: The saved tracking_states. + """ + self._in_flight_orders.update({ + key: CoinzoomInFlightOrder.from_json(value) + for key, value in saved_states.items() + }) + + def supported_order_types(self) -> List[OrderType]: + """ + :return a list of OrderType supported by this connector. + Note that Market order type is no longer required and will not be used. + """ + return [OrderType.LIMIT, OrderType.LIMIT_MAKER] + + def start(self, clock: Clock, timestamp: float): + """ + This function is called automatically by the clock. + """ + super().start(clock, timestamp) + + def stop(self, clock: Clock): + """ + This function is called automatically by the clock. + """ + super().stop(clock) + + async def start_network(self): + """ + This function is required by NetworkIterator base class and is called automatically. + It starts tracking order book, polling trading rules, + updating statuses and tracking user data. + """ + self._order_book_tracker.start() + self._trading_rules_polling_task = safe_ensure_future(self._trading_rules_polling_loop()) + if self._trading_required: + self._status_polling_task = safe_ensure_future(self._status_polling_loop()) + self._user_stream_tracker_task = safe_ensure_future(self._user_stream_tracker.start()) + self._user_stream_event_listener_task = safe_ensure_future(self._user_stream_event_listener()) + + async def stop_network(self): + """ + This function is required by NetworkIterator base class and is called automatically. + """ + self._order_book_tracker.stop() + if self._status_polling_task is not None: + self._status_polling_task.cancel() + self._status_polling_task = None + if self._trading_rules_polling_task is not None: + self._trading_rules_polling_task.cancel() + self._trading_rules_polling_task = None + if self._status_polling_task is not None: + self._status_polling_task.cancel() + self._status_polling_task = None + if self._user_stream_tracker_task is not None: + self._user_stream_tracker_task.cancel() + self._user_stream_tracker_task = None + if self._user_stream_event_listener_task is not None: + self._user_stream_event_listener_task.cancel() + self._user_stream_event_listener_task = None + + async def check_network(self) -> NetworkStatus: + """ + This function is required by NetworkIterator base class and is called periodically to check + the network connection. Simply ping the network (or call any light weight public API). + """ + try: + # since there is no ping endpoint, the lowest rate call is to get BTC-USD symbol + await self._api_request("GET", Constants.ENDPOINT['SYMBOL']) + except asyncio.CancelledError: + raise + except Exception: + return NetworkStatus.NOT_CONNECTED + return NetworkStatus.CONNECTED + + async def _http_client(self) -> aiohttp.ClientSession: + """ + :returns Shared client session instance + """ + if self._shared_client is None: + self._shared_client = aiohttp.ClientSession() + return self._shared_client + + async def _trading_rules_polling_loop(self): + """ + Periodically update trading rule. + """ + while True: + try: + await self._update_trading_rules() + await asyncio.sleep(Constants.INTERVAL_TRADING_RULES) + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().network(f"Unexpected error while fetching trading rules. Error: {str(e)}", + exc_info=True, + app_warning_msg=("Could not fetch new trading rules from " + f"{Constants.EXCHANGE_NAME}. Check network connection.")) + await asyncio.sleep(0.5) + + async def _update_trading_rules(self): + symbols_info = await self._api_request("GET", endpoint=Constants.ENDPOINT['SYMBOL']) + self._trading_rules.clear() + self._trading_rules = self._format_trading_rules(symbols_info) + + def _format_trading_rules(self, symbols_info: Dict[str, Any]) -> Dict[str, TradingRule]: + """ + Converts json API response into a dictionary of trading rules. + :param symbols_info: The json API response + :return A dictionary of trading rules. + Response Example: + [ + { + "symbol" : "BTC/USD", + "baseCurrencyCode" : "BTC", + "termCurrencyCode" : "USD", + "minTradeAmt" : 0.0001, + "maxTradeAmt" : 10, + "maxPricePrecision" : 2, + "maxQuantityPrecision" : 6, + "issueOnly" : false + } + ] + """ + result = {} + for rule in symbols_info: + try: + trading_pair = convert_from_exchange_trading_pair(rule["symbol"]) + min_amount = Decimal(str(rule["minTradeAmt"])) + min_price = Decimal(f"1e-{rule['maxPricePrecision']}") + result[trading_pair] = TradingRule(trading_pair, + min_order_size=min_amount, + max_order_size=Decimal(str(rule["maxTradeAmt"])), + min_price_increment=min_price, + min_base_amount_increment=min_amount, + min_notional_size=min(min_price * min_amount, Decimal("0.00000001")), + max_price_significant_digits=Decimal(str(rule["maxPricePrecision"])), + ) + except Exception: + self.logger().error(f"Error parsing the trading pair rule {rule}. Skipping.", exc_info=True) + return result + + async def _api_request(self, + method: str, + endpoint: str, + params: Optional[Dict[str, Any]] = None, + is_auth_required: bool = False, + try_count: int = 0) -> Dict[str, Any]: + """ + Sends an aiohttp request and waits for a response. + :param method: The HTTP method, e.g. get or post + :param endpoint: The path url or the API end point + :param params: Additional get/post parameters + :param is_auth_required: Whether an authentication is required, when True the function will add encrypted + signature to the request. + :returns A response in json format. + """ + async with self._throttler.weighted_task(request_weight=1): + url = f"{Constants.REST_URL}/{endpoint}" + shared_client = await self._http_client() + # Turn `params` into either GET params or POST body data + qs_params: dict = params if method.upper() == "GET" else None + req_params = ujson.dumps(params) if method.upper() == "POST" and params is not None else None + # Generate auth headers if needed. + headers: dict = {"Content-Type": "application/json", "User-Agent": "hummingbot"} + if is_auth_required: + headers: dict = self._coinzoom_auth.get_headers() + # Build request coro + response_coro = shared_client.request(method=method.upper(), url=url, headers=headers, + params=qs_params, data=req_params, + timeout=Constants.API_CALL_TIMEOUT) + http_status, parsed_response, request_errors = await aiohttp_response_with_errors(response_coro) + if request_errors or parsed_response is None: + if try_count < Constants.API_MAX_RETRIES: + try_count += 1 + time_sleep = retry_sleep_time(try_count) + self.logger().info(f"Error fetching data from {url}. HTTP status is {http_status}. " + f"Retrying in {time_sleep:.0f}s.") + await asyncio.sleep(time_sleep) + return await self._api_request(method=method, endpoint=endpoint, params=params, + is_auth_required=is_auth_required, try_count=try_count) + else: + raise CoinzoomAPIError({"error": parsed_response, "status": http_status}) + if "error" in parsed_response: + raise CoinzoomAPIError(parsed_response) + return parsed_response + + def get_order_price_quantum(self, trading_pair: str, price: Decimal): + """ + Returns a price step, a minimum price increment for a given trading pair. + """ + trading_rule = self._trading_rules[trading_pair] + return trading_rule.min_price_increment + + def get_order_size_quantum(self, trading_pair: str, order_size: Decimal): + """ + Returns an order amount step, a minimum amount increment for a given trading pair. + """ + trading_rule = self._trading_rules[trading_pair] + return Decimal(trading_rule.min_base_amount_increment) + + def get_order_book(self, trading_pair: str) -> OrderBook: + if trading_pair not in self._order_book_tracker.order_books: + raise ValueError(f"No order book exists for '{trading_pair}'.") + return self._order_book_tracker.order_books[trading_pair] + + def buy(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, + price: Decimal = s_decimal_NaN, **kwargs) -> str: + """ + Buys an amount of base asset (of the given trading pair). This function returns immediately. + To see an actual order, you'll have to wait for BuyOrderCreatedEvent. + :param trading_pair: The market (e.g. BTC-USDT) to buy from + :param amount: The amount in base token value + :param order_type: The order type + :param price: The price (note: this is no longer optional) + :returns A new internal order id + """ + order_id: str = get_new_client_order_id(True, trading_pair) + safe_ensure_future(self._create_order(TradeType.BUY, order_id, trading_pair, amount, order_type, price)) + return order_id + + def sell(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, + price: Decimal = s_decimal_NaN, **kwargs) -> str: + """ + Sells an amount of base asset (of the given trading pair). This function returns immediately. + To see an actual order, you'll have to wait for SellOrderCreatedEvent. + :param trading_pair: The market (e.g. BTC-USDT) to sell from + :param amount: The amount in base token value + :param order_type: The order type + :param price: The price (note: this is no longer optional) + :returns A new internal order id + """ + order_id: str = get_new_client_order_id(False, trading_pair) + safe_ensure_future(self._create_order(TradeType.SELL, order_id, trading_pair, amount, order_type, price)) + return order_id + + def cancel(self, trading_pair: str, order_id: str): + """ + Cancel an order. This function returns immediately. + To get the cancellation result, you'll have to wait for OrderCancelledEvent. + :param trading_pair: The market (e.g. BTC-USDT) of the order. + :param order_id: The internal order id (also called client_order_id) + """ + safe_ensure_future(self._execute_cancel(trading_pair, order_id)) + return order_id + + async def _create_order(self, + trade_type: TradeType, + order_id: str, + trading_pair: str, + amount: Decimal, + order_type: OrderType, + price: Decimal): + """ + Calls create-order API end point to place an order, starts tracking the order and triggers order created event. + :param trade_type: BUY or SELL + :param order_id: Internal order id (also called client_order_id) + :param trading_pair: The market to place order + :param amount: The order amount (in base token value) + :param order_type: The order type + :param price: The order price + """ + if not order_type.is_limit_type(): + raise Exception(f"Unsupported order type: {order_type}") + trading_rule = self._trading_rules[trading_pair] + + amount = self.quantize_order_amount(trading_pair, amount) + price = self.quantize_order_price(trading_pair, price) + if amount < trading_rule.min_order_size: + raise ValueError(f"Buy order amount {amount} is lower than the minimum order size " + f"{trading_rule.min_order_size}.") + order_type_str = order_type.name.upper().split("_")[0] + api_params = {"symbol": convert_to_exchange_trading_pair(trading_pair), + "orderType": order_type_str, + "orderSide": trade_type.name.upper(), + "quantity": f"{amount:f}", + "price": f"{price:f}", + "originType": Constants.HBOT_BROKER_ID, + # CoinZoom doesn't support client order id yet + # "clientOrderId": order_id, + "payFeesWithZoomToken": "true", + } + self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, order_type) + try: + order_result = await self._api_request("POST", Constants.ENDPOINT["ORDER_CREATE"], api_params, True) + exchange_order_id = str(order_result) + tracked_order = self._in_flight_orders.get(order_id) + if tracked_order is not None: + self.logger().info(f"Created {order_type.name} {trade_type.name} order {order_id} for " + f"{amount} {trading_pair}.") + tracked_order.update_exchange_order_id(exchange_order_id) + if trade_type is TradeType.BUY: + event_tag = MarketEvent.BuyOrderCreated + event_cls = BuyOrderCreatedEvent + else: + event_tag = MarketEvent.SellOrderCreated + event_cls = SellOrderCreatedEvent + self.trigger_event(event_tag, + event_cls(self.current_timestamp, order_type, trading_pair, amount, price, order_id)) + except asyncio.CancelledError: + raise + except CoinzoomAPIError as e: + error_reason = e.error_payload.get('error', {}).get('message') + self.stop_tracking_order(order_id) + self.logger().network( + f"Error submitting {trade_type.name} {order_type.name} order to {Constants.EXCHANGE_NAME} for " + f"{amount} {trading_pair} {price} - {error_reason}.", + exc_info=True, + app_warning_msg=(f"Error submitting order to {Constants.EXCHANGE_NAME} - {error_reason}.") + ) + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent(self.current_timestamp, order_id, order_type)) + + def start_tracking_order(self, + order_id: str, + exchange_order_id: str, + trading_pair: str, + trade_type: TradeType, + price: Decimal, + amount: Decimal, + order_type: OrderType): + """ + Starts tracking an order by simply adding it into _in_flight_orders dictionary. + """ + self._in_flight_orders[order_id] = CoinzoomInFlightOrder( + client_order_id=order_id, + exchange_order_id=exchange_order_id, + trading_pair=trading_pair, + order_type=order_type, + trade_type=trade_type, + price=price, + amount=amount + ) + + def stop_tracking_order(self, order_id: str): + """ + Stops tracking an order by simply removing it from _in_flight_orders dictionary. + """ + if order_id in self._in_flight_orders: + del self._in_flight_orders[order_id] + if order_id in self._order_not_found_records: + del self._order_not_found_records[order_id] + + async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: + """ + Executes order cancellation process by first calling cancel-order API. The API result doesn't confirm whether + the cancellation is successful, it simply states it receives the request. + :param trading_pair: The market trading pair (Unused during cancel on CoinZoom) + :param order_id: The internal order id + order.last_state to change to CANCELED + """ + order_was_cancelled = False + try: + tracked_order = self._in_flight_orders.get(order_id) + if tracked_order is None: + raise ValueError(f"Failed to cancel order - {order_id}. Order not found.") + if tracked_order.exchange_order_id is None: + await tracked_order.get_exchange_order_id() + ex_order_id = tracked_order.exchange_order_id + api_params = { + "orderId": ex_order_id, + "symbol": convert_to_exchange_trading_pair(trading_pair) + } + await self._api_request("POST", + Constants.ENDPOINT["ORDER_DELETE"], + api_params, + is_auth_required=True) + order_was_cancelled = True + except asyncio.CancelledError: + raise + except CoinzoomAPIError as e: + err = e.error_payload.get('error', e.error_payload) + self.logger().error(f"Order Cancel API Error: {err}") + # CoinZoom doesn't report any error if the order wasn't found so we can only handle API failures here. + self._order_not_found_records[order_id] = self._order_not_found_records.get(order_id, 0) + 1 + if self._order_not_found_records[order_id] >= self.ORDER_NOT_EXIST_CANCEL_COUNT: + order_was_cancelled = True + if order_was_cancelled: + self.logger().info(f"Successfully cancelled order {order_id} on {Constants.EXCHANGE_NAME}.") + self.stop_tracking_order(order_id) + self.trigger_event(MarketEvent.OrderCancelled, + OrderCancelledEvent(self.current_timestamp, order_id)) + tracked_order.cancelled_event.set() + return CancellationResult(order_id, True) + else: + self.logger().network( + f"Failed to cancel order {order_id}: {err.get('message', str(err))}", + exc_info=True, + app_warning_msg=f"Failed to cancel the order {order_id} on {Constants.EXCHANGE_NAME}. " + f"Check API key and network connection." + ) + return CancellationResult(order_id, False) + + async def _status_polling_loop(self): + """ + Periodically update user balances and order status via REST API. This serves as a fallback measure for web + socket API updates. + """ + while True: + try: + self._poll_notifier = asyncio.Event() + await self._poll_notifier.wait() + await safe_gather( + self._update_balances(), + self._update_order_status(), + ) + self._last_poll_timestamp = self.current_timestamp + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().error(str(e), exc_info=True) + warn_msg = (f"Could not fetch account updates from {Constants.EXCHANGE_NAME}. " + "Check API key and network connection.") + self.logger().network("Unexpected error while fetching account updates.", exc_info=True, + app_warning_msg=warn_msg) + await asyncio.sleep(0.5) + + async def _update_balances(self): + """ + Calls REST API to update total and available balances. + """ + account_info = await self._api_request("GET", Constants.ENDPOINT["USER_BALANCES"], is_auth_required=True) + self._process_balance_message(account_info) + + async def _update_order_status(self): + """ + Calls REST API to get status update for each in-flight order. + """ + last_tick = int(self._last_poll_timestamp / Constants.UPDATE_ORDER_STATUS_INTERVAL) + current_tick = int(self.current_timestamp / Constants.UPDATE_ORDER_STATUS_INTERVAL) + + if current_tick > last_tick and len(self._in_flight_orders) > 0: + tracked_orders = list(self._in_flight_orders.values()) + api_params = { + 'symbol': None, + 'orderSide': None, + 'orderStatuses': ["NEW", "PARTIALLY_FILLED"], + 'size': 500, + 'bookmarkOrderId': None + } + self.logger().debug(f"Polling for order status updates of {len(tracked_orders)} orders.") + open_orders = await self._api_request("POST", + Constants.ENDPOINT["ORDER_STATUS"], + api_params, + is_auth_required=True) + + open_orders_dict = {o['id']: o for o in open_orders} + found_ex_order_ids = list(open_orders_dict.keys()) + + for tracked_order in tracked_orders: + client_order_id = tracked_order.client_order_id + ex_order_id = tracked_order.exchange_order_id + if ex_order_id not in found_ex_order_ids: + self._order_not_found_records[client_order_id] = \ + self._order_not_found_records.get(client_order_id, 0) + 1 + if self._order_not_found_records[client_order_id] < self.ORDER_NOT_EXIST_CONFIRMATION_COUNT: + # Wait until the order is not found a few times before actually treating it as failed. + continue + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent( + self.current_timestamp, client_order_id, tracked_order.order_type)) + self.stop_tracking_order(client_order_id) + else: + self._process_order_message(open_orders_dict[ex_order_id]) + + def _process_order_message(self, order_msg: Dict[str, Any]): + """ + Updates in-flight order and triggers cancellation or failure event if needed. + :param order_msg: The order response from either REST or web socket API (they are of the same format) + Example Orders: + REST request + { + "id" : "977f82aa-23dc-4c8b-982c-2ee7d2002882", + "clientOrderId" : null, + "symbol" : "BTC/USD", + "orderType" : "LIMIT", + "orderSide" : "BUY", + "quantity" : 0.1, + "price" : 54570, + "payFeesWithZoomToken" : false, + "orderStatus" : "PARTIALLY_FILLED", + "timestamp" : "2021-03-24T04:07:26.260253Z", + "executions" : + [ + { + "id" : "38761582-2b37-4e27-a561-434981d21a96", + "executionType" : "PARTIAL_FILL", + "orderStatus" : "PARTIALLY_FILLED", + "lastPrice" : 54570, + "averagePrice" : 54570, + "lastQuantity" : 0.01, + "leavesQuantity" : 0.09, + "cumulativeQuantity" : 0.01, + "rejectReason" : null, + "timestamp" : "2021-03-24T04:07:44.503222Z" + } + ] + } + WS request + { + 'orderId': '962a2a54-fbcf-4d89-8f37-a8854020a823', + 'symbol': 'BTC/USD', 'orderType': 'LIMIT', + 'orderSide': 'BUY', + 'price': 5000, + 'quantity': 0.001, + 'executionType': 'CANCEL', + 'orderStatus': 'CANCELLED', + 'lastQuantity': 0, + 'leavesQuantity': 0, + 'cumulativeQuantity': 0, + 'transactTime': '2021-03-23T19:06:51.155520Z' + + ... Optional fields + + 'id': '4eb3f26c-91bd-4bd2-bacb-15b2f432c452', + "orderType": "LIMIT", + "lastPrice": 56518.7, + "averagePrice": 56518.7, + } + """ + # Looks like CoinZoom might support clientOrderId eventually so leaving this here for now. + # if order_msg.get('clientOrderId') is not None: + # client_order_id = order_msg["clientOrderId"] + # if client_order_id not in self._in_flight_orders: + # return + # tracked_order = self._in_flight_orders[client_order_id] + # else: + if "orderId" not in order_msg: + exchange_order_id = str(order_msg["id"]) + else: + exchange_order_id = str(order_msg["orderId"]) + tracked_orders = list(self._in_flight_orders.values()) + track_order = [o for o in tracked_orders if exchange_order_id == o.exchange_order_id] + if not track_order: + return + tracked_order = track_order[0] + + # Estimate fee + order_msg["trade_fee"] = self.estimate_fee_pct(tracked_order.order_type is OrderType.LIMIT_MAKER) + updated = tracked_order.update_with_order_update(order_msg) + # Call Update balances on every message to catch order create, fill and cancel. + safe_ensure_future(self._update_balances()) + + if updated: + safe_ensure_future(self._trigger_order_fill(tracked_order, order_msg)) + elif tracked_order.is_cancelled: + self.logger().info(f"Successfully cancelled order {tracked_order.client_order_id}.") + self.stop_tracking_order(tracked_order.client_order_id) + self.trigger_event(MarketEvent.OrderCancelled, + OrderCancelledEvent(self.current_timestamp, tracked_order.client_order_id)) + tracked_order.cancelled_event.set() + elif tracked_order.is_failure: + self.logger().info(f"The order {tracked_order.client_order_id} has failed according to order status API. ") + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent( + self.current_timestamp, tracked_order.client_order_id, tracked_order.order_type)) + self.stop_tracking_order(tracked_order.client_order_id) + + async def _trigger_order_fill(self, + tracked_order: CoinzoomInFlightOrder, + update_msg: Dict[str, Any]): + self.trigger_event( + MarketEvent.OrderFilled, + OrderFilledEvent( + self.current_timestamp, + tracked_order.client_order_id, + tracked_order.trading_pair, + tracked_order.trade_type, + tracked_order.order_type, + Decimal(str(update_msg.get("averagePrice", update_msg.get("price", "0")))), + tracked_order.executed_amount_base, + TradeFee(percent=update_msg["trade_fee"]), + update_msg.get("exchange_trade_id", update_msg.get("id", update_msg.get("orderId"))) + ) + ) + if math.isclose(tracked_order.executed_amount_base, tracked_order.amount) or \ + tracked_order.executed_amount_base >= tracked_order.amount or \ + tracked_order.is_done: + tracked_order.last_state = "FILLED" + self.logger().info(f"The {tracked_order.trade_type.name} order " + f"{tracked_order.client_order_id} has completed " + f"according to order status API.") + event_tag = MarketEvent.BuyOrderCompleted if tracked_order.trade_type is TradeType.BUY \ + else MarketEvent.SellOrderCompleted + event_class = BuyOrderCompletedEvent if tracked_order.trade_type is TradeType.BUY \ + else SellOrderCompletedEvent + await asyncio.sleep(0.1) + self.trigger_event(event_tag, + event_class(self.current_timestamp, + tracked_order.client_order_id, + tracked_order.base_asset, + tracked_order.quote_asset, + tracked_order.fee_asset, + tracked_order.executed_amount_base, + tracked_order.executed_amount_quote, + tracked_order.fee_paid, + tracked_order.order_type)) + self.stop_tracking_order(tracked_order.client_order_id) + + def _process_balance_message(self, balance_update): + local_asset_names = set(self._account_balances.keys()) + remote_asset_names = set() + for account in balance_update: + asset_name = account["currency"] + total_bal = Decimal(str(account["totalBalance"])) + self._account_available_balances[asset_name] = total_bal + Decimal(str(account["reservedBalance"])) + self._account_balances[asset_name] = total_bal + remote_asset_names.add(asset_name) + + asset_names_to_remove = local_asset_names.difference(remote_asset_names) + for asset_name in asset_names_to_remove: + del self._account_available_balances[asset_name] + del self._account_balances[asset_name] + + async def cancel_all(self, timeout_seconds: float) -> List[CancellationResult]: + """ + Cancels all in-flight orders and waits for cancellation results. + Used by bot's top level stop and exit commands (cancelling outstanding orders on exit) + :param timeout_seconds: The timeout at which the operation will be canceled. + :returns List of CancellationResult which indicates whether each order is successfully cancelled. + """ + # if self._trading_pairs is None: + # raise Exception("cancel_all can only be used when trading_pairs are specified.") + open_orders = [o for o in self._in_flight_orders.values() if not o.is_done] + if len(open_orders) == 0: + return [] + tasks = [self._execute_cancel(o.trading_pair, o.client_order_id) for o in open_orders] + cancellation_results = [] + try: + async with timeout(timeout_seconds): + cancellation_results = await safe_gather(*tasks, return_exceptions=False) + except Exception: + self.logger().network( + "Unexpected error cancelling orders.", exc_info=True, + app_warning_msg=(f"Failed to cancel all orders on {Constants.EXCHANGE_NAME}. " + "Check API key and network connection.") + ) + return cancellation_results + + def tick(self, timestamp: float): + """ + Is called automatically by the clock for each clock's tick (1 second by default). + It checks if status polling task is due for execution. + """ + now = time.time() + poll_interval = (Constants.SHORT_POLL_INTERVAL + if now - self._user_stream_tracker.last_recv_time > 60.0 + else Constants.LONG_POLL_INTERVAL) + last_tick = int(self._last_timestamp / poll_interval) + current_tick = int(timestamp / poll_interval) + if current_tick > last_tick: + if not self._poll_notifier.is_set(): + self._poll_notifier.set() + self._last_timestamp = timestamp + + def get_fee(self, + base_currency: str, + quote_currency: str, + order_type: OrderType, + order_side: TradeType, + amount: Decimal, + price: Decimal = s_decimal_NaN) -> TradeFee: + """ + To get trading fee, this function is simplified by using fee override configuration. Most parameters to this + function are ignore except order_type. Use OrderType.LIMIT_MAKER to specify you want trading fee for + maker order. + """ + is_maker = order_type is OrderType.LIMIT_MAKER + return TradeFee(percent=self.estimate_fee_pct(is_maker)) + + async def _iter_user_event_queue(self) -> AsyncIterable[Dict[str, any]]: + while True: + try: + yield await self._user_stream_tracker.user_stream.get() + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unknown error. Retrying after 1 seconds.", exc_info=True, + app_warning_msg=(f"Could not fetch user events from {Constants.EXCHANGE_NAME}. " + "Check API key and network connection.")) + await asyncio.sleep(1.0) + + async def _user_stream_event_listener(self): + """ + Listens to message in _user_stream_tracker.user_stream queue. The messages are put in by + CoinzoomAPIUserStreamDataSource. + """ + async for event_message in self._iter_user_event_queue(): + try: + event_methods = [ + Constants.WS_METHODS["USER_ORDERS"], + Constants.WS_METHODS["USER_ORDERS_CANCEL"], + ] + + msg_keys = list(event_message.keys()) if event_message is not None else [] + + method_key = [key for key in msg_keys if key in event_methods] + + if len(method_key) != 1: + continue + + method: str = method_key[0] + + if method == Constants.WS_METHODS["USER_ORDERS"]: + self._process_order_message(event_message[method]) + elif method == Constants.WS_METHODS["USER_ORDERS_CANCEL"]: + self._process_order_message(event_message[method]) + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error in user stream listener loop.", exc_info=True) + await asyncio.sleep(5.0) + + # This is currently unused, but looks like a future addition. + async def get_open_orders(self) -> List[OpenOrder]: + tracked_orders = list(self._in_flight_orders.values()) + api_params = { + 'symbol': None, + 'orderSide': None, + 'orderStatuses': ["NEW", "PARTIALLY_FILLED"], + 'size': 500, + 'bookmarkOrderId': None + } + result = await self._api_request("POST", Constants.ENDPOINT["USER_ORDERS"], api_params, is_auth_required=True) + ret_val = [] + for order in result: + exchange_order_id = str(order["id"]) + # CoinZoom doesn't support client order ids yet so we must find it from the tracked orders. + track_order = [o for o in tracked_orders if exchange_order_id == o.exchange_order_id] + if not track_order or len(track_order) < 1: + # Skip untracked orders + continue + client_order_id = track_order[0].client_order_id + # if Constants.HBOT_BROKER_ID not in order["clientOrderId"]: + # continue + if order["orderType"] != OrderType.LIMIT.name.upper(): + self.logger().info(f"Unsupported order type found: {order['type']}") + # Skip and report non-limit orders + continue + ret_val.append( + OpenOrder( + client_order_id=client_order_id, + trading_pair=convert_from_exchange_trading_pair(order["symbol"]), + price=Decimal(str(order["price"])), + amount=Decimal(str(order["quantity"])), + executed_amount=Decimal(str(order["cumQuantity"])), + status=order["orderStatus"], + order_type=OrderType.LIMIT, + is_buy=True if order["orderSide"].lower() == TradeType.BUY.name.lower() else False, + time=str_date_to_ts(order["timestamp"]), + exchange_order_id=order["id"] + ) + ) + return ret_val diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py b/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py new file mode 100644 index 0000000000..61da8fdb0b --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py @@ -0,0 +1,163 @@ +from decimal import Decimal +from typing import ( + Any, + Dict, + Optional, +) +import asyncio +from hummingbot.core.event.events import ( + OrderType, + TradeType +) +from hummingbot.connector.in_flight_order_base import InFlightOrderBase + +s_decimal_0 = Decimal(0) + + +class CoinzoomInFlightOrder(InFlightOrderBase): + def __init__(self, + client_order_id: str, + exchange_order_id: Optional[str], + trading_pair: str, + order_type: OrderType, + trade_type: TradeType, + price: Decimal, + amount: Decimal, + initial_state: str = "NEW"): + super().__init__( + client_order_id, + exchange_order_id, + trading_pair, + order_type, + trade_type, + price, + amount, + initial_state, + ) + self.trade_id_set = set() + self.cancelled_event = asyncio.Event() + + @property + def is_done(self) -> bool: + return self.last_state in {"FILLED", "CANCELLED", "REJECTED"} + + @property + def is_failure(self) -> bool: + return self.last_state in {"REJECTED"} + + @property + def is_cancelled(self) -> bool: + return self.last_state in {"CANCELLED"} + + @classmethod + def from_json(cls, data: Dict[str, Any]) -> InFlightOrderBase: + """ + :param data: json data from API + :return: formatted InFlightOrder + """ + retval = CoinzoomInFlightOrder( + data["client_order_id"], + data["exchange_order_id"], + data["trading_pair"], + getattr(OrderType, data["order_type"]), + getattr(TradeType, data["trade_type"]), + Decimal(data["price"]), + Decimal(data["amount"]), + data["last_state"] + ) + retval.executed_amount_base = Decimal(data["executed_amount_base"]) + retval.executed_amount_quote = Decimal(data["executed_amount_quote"]) + retval.fee_asset = data["fee_asset"] + retval.fee_paid = Decimal(data["fee_paid"]) + retval.last_state = data["last_state"] + return retval + + def update_with_order_update(self, order_update: Dict[str, Any]) -> bool: + """ + Updates the in flight order with order update (from private/get-order-detail end point) + return: True if the order gets updated otherwise False + Example Orders: + REST request + { + "id" : "977f82aa-23dc-4c8b-982c-2ee7d2002882", + "clientOrderId" : null, + "symbol" : "BTC/USD", + "orderType" : "LIMIT", + "orderSide" : "BUY", + "quantity" : 0.1, + "price" : 54570, + "payFeesWithZoomToken" : false, + "orderStatus" : "PARTIALLY_FILLED", + "timestamp" : "2021-03-24T04:07:26.260253Z", + "executions" : + [ + { + "id" : "38761582-2b37-4e27-a561-434981d21a96", + "executionType" : "PARTIAL_FILL", + "orderStatus" : "PARTIALLY_FILLED", + "lastPrice" : 54570, + "averagePrice" : 54570, + "lastQuantity" : 0.01, + "leavesQuantity" : 0.09, + "cumulativeQuantity" : 0.01, + "rejectReason" : null, + "timestamp" : "2021-03-24T04:07:44.503222Z" + } + ] + } + WS request + { + 'id': '4eb3f26c-91bd-4bd2-bacb-15b2f432c452', + 'orderId': '962a2a54-fbcf-4d89-8f37-a8854020a823', + 'symbol': 'BTC/USD', 'orderType': 'LIMIT', + 'orderSide': 'BUY', + 'price': 5000, + 'quantity': 0.001, + 'executionType': 'CANCEL', + 'orderStatus': 'CANCELLED', + 'lastQuantity': 0, + 'leavesQuantity': 0, + 'cumulativeQuantity': 0, + 'transactTime': '2021-03-23T19:06:51.155520Z' + } + """ + # Update order execution status + self.last_state = order_update["orderStatus"] + + if 'cumulativeQuantity' not in order_update and 'executions' not in order_update: + return False + + trades = order_update.get('executions') + if trades is not None: + new_trades = False + for trade in trades: + trade_id = str(trade["timestamp"]) + if trade_id not in self.trade_id_set: + self.trade_id_set.add(trade_id) + order_update["exchange_trade_id"] = trade.get("id") + # Add executed amounts + executed_price = Decimal(str(trade.get("lastPrice", "0"))) + self.executed_amount_base += Decimal(str(trade["lastQuantity"])) + self.executed_amount_quote += executed_price * self.executed_amount_base + # Set new trades flag + new_trades = True + if not new_trades: + # trades already recorded + return False + else: + trade_id = str(order_update["transactTime"]) + if trade_id in self.trade_id_set: + # trade already recorded + return False + self.trade_id_set.add(trade_id) + # Set executed amounts + executed_price = Decimal(str(order_update.get("averagePrice", order_update.get("price", "0")))) + self.executed_amount_base = Decimal(str(order_update["cumulativeQuantity"])) + self.executed_amount_quote = executed_price * self.executed_amount_base + if self.executed_amount_base <= s_decimal_0: + # No trades executed yet. + return False + self.fee_paid += order_update.get("trade_fee") * self.executed_amount_base + if not self.fee_asset: + self.fee_asset = self.quote_asset + return True diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book.py b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book.py new file mode 100644 index 0000000000..e771d48cf3 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python + +import logging +from hummingbot.connector.exchange.coinzoom.coinzoom_constants import Constants + +from sqlalchemy.engine import RowProxy +from typing import ( + Optional, + Dict, + List, Any) +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_message import ( + OrderBookMessage, OrderBookMessageType +) +from hummingbot.connector.exchange.coinzoom.coinzoom_order_book_message import CoinzoomOrderBookMessage +from .coinzoom_utils import ( + convert_from_exchange_trading_pair, + str_date_to_ts, +) + +_logger = None + + +class CoinzoomOrderBook(OrderBook): + @classmethod + def logger(cls) -> HummingbotLogger: + global _logger + if _logger is None: + _logger = logging.getLogger(__name__) + return _logger + + @classmethod + def snapshot_message_from_exchange(cls, + msg: Dict[str, any], + timestamp: float, + metadata: Optional[Dict] = None): + """ + Convert json snapshot data into standard OrderBookMessage format + :param msg: json snapshot data from live web socket stream + :param timestamp: timestamp attached to incoming data + :return: CoinzoomOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + return CoinzoomOrderBookMessage( + message_type=OrderBookMessageType.SNAPSHOT, + content=msg, + timestamp=timestamp + ) + + @classmethod + def snapshot_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of snapshot data into standard OrderBookMessage format + :param record: a row of snapshot data from the database + :return: CoinzoomOrderBookMessage + """ + return CoinzoomOrderBookMessage( + message_type=OrderBookMessageType.SNAPSHOT, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def diff_message_from_exchange(cls, + msg: Dict[str, any], + timestamp: Optional[float] = None, + metadata: Optional[Dict] = None): + """ + Convert json diff data into standard OrderBookMessage format + :param msg: json diff data from live web socket stream + :param timestamp: timestamp attached to incoming data + :return: CoinzoomOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + return CoinzoomOrderBookMessage( + message_type=OrderBookMessageType.DIFF, + content=msg, + timestamp=timestamp + ) + + @classmethod + def diff_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of diff data into standard OrderBookMessage format + :param record: a row of diff data from the database + :return: CoinzoomOrderBookMessage + """ + return CoinzoomOrderBookMessage( + message_type=OrderBookMessageType.DIFF, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def trade_message_from_exchange(cls, + msg: Dict[str, Any], + timestamp: Optional[float] = None, + metadata: Optional[Dict] = None): + """ + Convert a trade data into standard OrderBookMessage format + :param record: a trade data from the database + :return: CoinzoomOrderBookMessage + """ + + trade_msg = { + "trade_type": msg[4], + "price": msg[1], + "amount": msg[2], + "trading_pair": convert_from_exchange_trading_pair(msg[0]) + } + trade_timestamp = str_date_to_ts(msg[3]) + + return CoinzoomOrderBookMessage( + message_type=OrderBookMessageType.TRADE, + content=trade_msg, + timestamp=trade_timestamp + ) + + @classmethod + def trade_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of trade data into standard OrderBookMessage format + :param record: a row of trade data from the database + :return: CoinzoomOrderBookMessage + """ + return CoinzoomOrderBookMessage( + message_type=OrderBookMessageType.TRADE, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def from_snapshot(cls, snapshot: OrderBookMessage): + raise NotImplementedError(Constants.EXCHANGE_NAME + " order book needs to retain individual order data.") + + @classmethod + def restore_from_snapshot_and_diffs(self, snapshot: OrderBookMessage, diffs: List[OrderBookMessage]): + raise NotImplementedError(Constants.EXCHANGE_NAME + " order book needs to retain individual order data.") diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py new file mode 100644 index 0000000000..d6bc00541d --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +from typing import ( + Dict, + Optional, +) + +from hummingbot.core.data_type.order_book_message import ( + OrderBookMessage, + OrderBookMessageType, +) +from hummingbot.connector.exchange.coinzoom.coinzoom_constants import Constants + + +class CoinzoomOrderBookMessage(OrderBookMessage): + def __new__( + cls, + message_type: OrderBookMessageType, + content: Dict[str, any], + timestamp: Optional[float] = None, + *args, + **kwargs, + ): + if timestamp is None: + if message_type is OrderBookMessageType.SNAPSHOT: + raise ValueError("timestamp must not be None when initializing snapshot messages.") + timestamp = content["timestamp"] + + return super(CoinzoomOrderBookMessage, cls).__new__( + cls, message_type, content, timestamp=timestamp, *args, **kwargs + ) + + @property + def update_id(self) -> int: + if self.type in [OrderBookMessageType.DIFF, OrderBookMessageType.SNAPSHOT]: + return self.timestamp + else: + return -1 + + @property + def trade_id(self) -> int: + if self.type is OrderBookMessageType.TRADE: + return self.timestamp + return -1 + + @property + def trading_pair(self) -> str: + return self.content["trading_pair"] + + # The `asks` and `bids` properties are only used in the methods below. + # They are all replaced or unused in this connector: + # OrderBook.restore_from_snapshot_and_diffs + # OrderBookTracker._track_single_book + # MockAPIOrderBookDataSource.get_tracking_pairs + @property + def asks(self): + raise NotImplementedError(Constants.EXCHANGE_NAME + " order book uses active_order_tracker.") + + @property + def bids(self): + raise NotImplementedError(Constants.EXCHANGE_NAME + " order book uses active_order_tracker.") + + def __eq__(self, other) -> bool: + return self.type == other.type and self.timestamp == other.timestamp + + def __lt__(self, other) -> bool: + if self.timestamp != other.timestamp: + return self.timestamp < other.timestamp + else: + """ + If timestamp is the same, the ordering is snapshot < diff < trade + """ + return self.type.value < other.type.value diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker.py b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker.py new file mode 100644 index 0000000000..c81ca9a7bd --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +import asyncio +import bisect +import logging +from hummingbot.connector.exchange.coinzoom.coinzoom_constants import Constants +import time + +from collections import defaultdict, deque +from typing import Optional, Dict, List, Deque +from hummingbot.core.data_type.order_book_message import OrderBookMessageType +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.order_book_tracker import OrderBookTracker +from hummingbot.connector.exchange.coinzoom.coinzoom_order_book_message import CoinzoomOrderBookMessage +from hummingbot.connector.exchange.coinzoom.coinzoom_active_order_tracker import CoinzoomActiveOrderTracker +from hummingbot.connector.exchange.coinzoom.coinzoom_api_order_book_data_source import CoinzoomAPIOrderBookDataSource +from hummingbot.connector.exchange.coinzoom.coinzoom_order_book import CoinzoomOrderBook + + +class CoinzoomOrderBookTracker(OrderBookTracker): + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, trading_pairs: Optional[List[str]] = None,): + super().__init__(CoinzoomAPIOrderBookDataSource(trading_pairs), trading_pairs) + + self._ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + self._order_book_snapshot_stream: asyncio.Queue = asyncio.Queue() + self._order_book_diff_stream: asyncio.Queue = asyncio.Queue() + self._order_book_trade_stream: asyncio.Queue = asyncio.Queue() + self._process_msg_deque_task: Optional[asyncio.Task] = None + self._past_diffs_windows: Dict[str, Deque] = {} + self._order_books: Dict[str, CoinzoomOrderBook] = {} + self._saved_message_queues: Dict[str, Deque[CoinzoomOrderBookMessage]] = \ + defaultdict(lambda: deque(maxlen=1000)) + self._active_order_trackers: Dict[str, CoinzoomActiveOrderTracker] = defaultdict(CoinzoomActiveOrderTracker) + self._order_book_stream_listener_task: Optional[asyncio.Task] = None + self._order_book_trade_listener_task: Optional[asyncio.Task] = None + + @property + def exchange_name(self) -> str: + """ + Name of the current exchange + """ + return Constants.EXCHANGE_NAME + + async def _track_single_book(self, trading_pair: str): + """ + Update an order book with changes from the latest batch of received messages + """ + past_diffs_window: Deque[CoinzoomOrderBookMessage] = deque() + self._past_diffs_windows[trading_pair] = past_diffs_window + + message_queue: asyncio.Queue = self._tracking_message_queues[trading_pair] + order_book: CoinzoomOrderBook = self._order_books[trading_pair] + active_order_tracker: CoinzoomActiveOrderTracker = self._active_order_trackers[trading_pair] + + last_message_timestamp: float = time.time() + diff_messages_accepted: int = 0 + + while True: + try: + message: CoinzoomOrderBookMessage = None + saved_messages: Deque[CoinzoomOrderBookMessage] = self._saved_message_queues[trading_pair] + # Process saved messages first if there are any + if len(saved_messages) > 0: + message = saved_messages.popleft() + else: + message = await message_queue.get() + + if message.type is OrderBookMessageType.DIFF: + bids, asks = active_order_tracker.convert_diff_message_to_order_book_row(message) + order_book.apply_diffs(bids, asks, message.update_id) + past_diffs_window.append(message) + while len(past_diffs_window) > self.PAST_DIFF_WINDOW_SIZE: + past_diffs_window.popleft() + diff_messages_accepted += 1 + + # Output some statistics periodically. + now: float = time.time() + if int(now / 60.0) > int(last_message_timestamp / 60.0): + self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") + diff_messages_accepted = 0 + last_message_timestamp = now + elif message.type is OrderBookMessageType.SNAPSHOT: + past_diffs: List[CoinzoomOrderBookMessage] = list(past_diffs_window) + # only replay diffs later than snapshot, first update active order with snapshot then replay diffs + replay_position = bisect.bisect_right(past_diffs, message) + replay_diffs = past_diffs[replay_position:] + s_bids, s_asks = active_order_tracker.convert_snapshot_message_to_order_book_row(message) + order_book.apply_snapshot(s_bids, s_asks, message.update_id) + for diff_message in replay_diffs: + d_bids, d_asks = active_order_tracker.convert_diff_message_to_order_book_row(diff_message) + order_book.apply_diffs(d_bids, d_asks, diff_message.update_id) + + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + f"Unexpected error processing order book messages for {trading_pair}.", + exc_info=True, + app_warning_msg="Unexpected error processing order book messages. Retrying after 5 seconds." + ) + await asyncio.sleep(5.0) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker_entry.py b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker_entry.py new file mode 100644 index 0000000000..94feda5275 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker_entry.py @@ -0,0 +1,21 @@ +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_tracker_entry import OrderBookTrackerEntry +from hummingbot.connector.exchange.coinzoom.coinzoom_active_order_tracker import CoinzoomActiveOrderTracker + + +class CoinzoomOrderBookTrackerEntry(OrderBookTrackerEntry): + def __init__( + self, trading_pair: str, timestamp: float, order_book: OrderBook, active_order_tracker: CoinzoomActiveOrderTracker + ): + self._active_order_tracker = active_order_tracker + super(CoinzoomOrderBookTrackerEntry, self).__init__(trading_pair, timestamp, order_book) + + def __repr__(self) -> str: + return ( + f"CoinzoomOrderBookTrackerEntry(trading_pair='{self._trading_pair}', timestamp='{self._timestamp}', " + f"order_book='{self._order_book}')" + ) + + @property + def active_order_tracker(self) -> CoinzoomActiveOrderTracker: + return self._active_order_tracker diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_user_stream_tracker.py b/hummingbot/connector/exchange/coinzoom/coinzoom_user_stream_tracker.py new file mode 100644 index 0000000000..79c7584d70 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_user_stream_tracker.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +import asyncio +import logging +from typing import ( + Optional, + List, +) +from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.user_stream_tracker import ( + UserStreamTracker +) +from hummingbot.core.utils.async_utils import ( + safe_ensure_future, + safe_gather, +) +from hummingbot.connector.exchange.coinzoom.coinzoom_api_user_stream_data_source import \ + CoinzoomAPIUserStreamDataSource +from hummingbot.connector.exchange.coinzoom.coinzoom_auth import CoinzoomAuth +from hummingbot.connector.exchange.coinzoom.coinzoom_constants import Constants + + +class CoinzoomUserStreamTracker(UserStreamTracker): + _cbpust_logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._bust_logger is None: + cls._bust_logger = logging.getLogger(__name__) + return cls._bust_logger + + def __init__(self, + coinzoom_auth: Optional[CoinzoomAuth] = None, + trading_pairs: Optional[List[str]] = []): + super().__init__() + self._coinzoom_auth: CoinzoomAuth = coinzoom_auth + self._trading_pairs: List[str] = trading_pairs + self._ev_loop: asyncio.events.AbstractEventLoop = asyncio.get_event_loop() + self._data_source: Optional[UserStreamTrackerDataSource] = None + self._user_stream_tracking_task: Optional[asyncio.Task] = None + + @property + def data_source(self) -> UserStreamTrackerDataSource: + """ + *required + Initializes a user stream data source (user specific order diffs from live socket stream) + :return: OrderBookTrackerDataSource + """ + if not self._data_source: + self._data_source = CoinzoomAPIUserStreamDataSource( + coinzoom_auth=self._coinzoom_auth, + trading_pairs=self._trading_pairs + ) + return self._data_source + + @property + def exchange_name(self) -> str: + """ + *required + Name of the current exchange + """ + return Constants.EXCHANGE_NAME + + async def start(self): + """ + *required + Start all listeners and tasks + """ + self._user_stream_tracking_task = safe_ensure_future( + self.data_source.listen_for_user_stream(self._ev_loop, self._user_stream) + ) + await safe_gather(self._user_stream_tracking_task) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py new file mode 100644 index 0000000000..498ed56541 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py @@ -0,0 +1,149 @@ +import aiohttp +import asyncio +import random +from dateutil.parser import parse as dateparse +from typing import ( + Any, + Dict, + Optional, +) + +from hummingbot.core.utils.tracking_nonce import get_tracking_nonce +from hummingbot.client.config.config_var import ConfigVar +from hummingbot.client.config.config_methods import using_exchange +from .coinzoom_constants import Constants + + +CENTRALIZED = True + +EXAMPLE_PAIR = "BTC-USD" + +DEFAULT_FEES = [0.2, 0.26] + + +class CoinzoomAPIError(IOError): + def __init__(self, error_payload: Dict[str, Any]): + super().__init__(str(error_payload)) + self.error_payload = error_payload + + +# convert date string to timestamp +def str_date_to_ts(date: str) -> int: + return int(dateparse(date).timestamp() * 1e3) + + +# Request ID class +class RequestId: + """ + Generate request ids + """ + _request_id: int = 0 + + @classmethod + def generate_request_id(cls) -> int: + return get_tracking_nonce() + + +def convert_from_exchange_trading_pair(ex_trading_pair: str) -> Optional[str]: + # CoinZoom uses uppercase (BTC/USDT) + return ex_trading_pair.replace("/", "-") + + +def convert_to_exchange_trading_pair(hb_trading_pair: str, alternative: bool = False) -> str: + # CoinZoom uses uppercase (BTCUSDT) + if alternative: + return hb_trading_pair.replace("-", "_").upper() + else: + return hb_trading_pair.replace("-", "/").upper() + + +def get_new_client_order_id(is_buy: bool, trading_pair: str) -> str: + side = "B" if is_buy else "S" + symbols = trading_pair.split("-") + base = symbols[0].upper() + quote = symbols[1].upper() + base_str = f"{base[0]}{base[-1]}" + quote_str = f"{quote[0]}{quote[-1]}" + return f"{Constants.HBOT_BROKER_ID}{side}{base_str}{quote_str}{get_tracking_nonce()}" + + +def retry_sleep_time(try_count: int) -> float: + random.seed() + randSleep = 1 + float(random.randint(1, 10) / 100) + return float(2 + float(randSleep * (1 + (try_count ** try_count)))) + + +async def aiohttp_response_with_errors(request_coroutine): + http_status, parsed_response, request_errors = None, None, False + try: + async with request_coroutine as response: + http_status = response.status + try: + parsed_response = await response.json() + except Exception: + if response.status not in [204]: + request_errors = True + try: + parsed_response = str(await response.read()) + if len(parsed_response) > 100: + parsed_response = f"{parsed_response[:100]} ... (truncated)" + except Exception: + pass + TempFailure = (parsed_response is None or + (response.status not in [200, 201, 204] and "error" not in parsed_response)) + if TempFailure: + parsed_response = response.reason if parsed_response is None else parsed_response + request_errors = True + except Exception: + request_errors = True + return http_status, parsed_response, request_errors + + +async def api_call_with_retries(method, + endpoint, + params: Optional[Dict[str, Any]] = None, + shared_client=None, + try_count: int = 0) -> Dict[str, Any]: + url = f"{Constants.REST_URL}/{endpoint}" + headers = {"Content-Type": "application/json", "User-Agent": "hummingbot"} + http_client = shared_client if shared_client is not None else aiohttp.ClientSession() + # Build request coro + response_coro = http_client.request(method=method.upper(), url=url, headers=headers, + params=params, timeout=Constants.API_CALL_TIMEOUT) + http_status, parsed_response, request_errors = await aiohttp_response_with_errors(response_coro) + if shared_client is None: + await http_client.close() + if request_errors or parsed_response is None: + if try_count < Constants.API_MAX_RETRIES: + try_count += 1 + time_sleep = retry_sleep_time(try_count) + print(f"Error fetching data from {url}. HTTP status is {http_status}. " + f"Retrying in {time_sleep:.0f}s.") + await asyncio.sleep(time_sleep) + return await api_call_with_retries(method=method, endpoint=endpoint, params=params, + shared_client=shared_client, try_count=try_count) + else: + raise CoinzoomAPIError({"error": parsed_response, "status": http_status}) + return parsed_response + + +KEYS = { + "coinzoom_api_key": + ConfigVar(key="coinzoom_api_key", + prompt=f"Enter your {Constants.EXCHANGE_NAME} API key >>> ", + required_if=using_exchange("coinzoom"), + is_secure=True, + is_connect_key=True), + "coinzoom_secret_key": + ConfigVar(key="coinzoom_secret_key", + prompt=f"Enter your {Constants.EXCHANGE_NAME} secret key >>> ", + required_if=using_exchange("coinzoom"), + is_secure=True, + is_connect_key=True), + "coinzoom_username": + ConfigVar(key="coinzoom_username", + prompt=f"Enter your {Constants.EXCHANGE_NAME} ZoomMe username >>> ", + required_if=using_exchange("coinzoom"), + is_secure=True, + is_connect_key=True), +} diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py b/hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py new file mode 100644 index 0000000000..7da31a20e4 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python +import asyncio +import logging +import websockets +import json +from hummingbot.connector.exchange.coinzoom.coinzoom_constants import Constants + + +from typing import ( + Any, + AsyncIterable, + Dict, + List, + Optional, +) +from websockets.exceptions import ConnectionClosed +from hummingbot.logger import HummingbotLogger +from hummingbot.connector.exchange.coinzoom.coinzoom_auth import CoinzoomAuth + +# reusable websocket class +# ToDo: We should eventually remove this class, and instantiate web socket connection normally (see Binance for example) + + +class CoinzoomWebsocket(): + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, + auth: Optional[CoinzoomAuth] = None): + self._auth: Optional[CoinzoomAuth] = auth + self._isPrivate = True if self._auth is not None else False + self._WS_URL = Constants.WS_PRIVATE_URL if self._isPrivate else Constants.WS_PUBLIC_URL + self._client: Optional[websockets.WebSocketClientProtocol] = None + self._is_subscribed = False + + @property + def is_subscribed(self): + return self._is_subscribed + + # connect to exchange + async def connect(self): + # if auth class was passed into websocket class + # we need to emit authenticated requests + extra_headers = self._auth.get_headers() if self._isPrivate else {"User-Agent": "hummingbot"} + self._client = await websockets.connect(self._WS_URL, extra_headers=extra_headers) + + return self._client + + # disconnect from exchange + async def disconnect(self): + if self._client is None: + return + + await self._client.close() + + # receive & parse messages + async def _messages(self) -> AsyncIterable[Any]: + try: + while True: + try: + raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=Constants.MESSAGE_TIMEOUT) + try: + msg = json.loads(raw_msg_str) + + # CoinZoom doesn't support ping or heartbeat messages. + # Can handle them here if that changes - use `safe_ensure_future`. + + # Check response for a subscribed/unsubscribed message; + result: List[str] = list([d['result'] + for k, d in msg.items() + if (isinstance(d, dict) and d.get('result') is not None)]) + + if len(result): + if result[0] == 'subscribed': + self._is_subscribed = True + elif result[0] == 'unsubscribed': + self._is_subscribed = False + yield None + else: + yield msg + except ValueError: + continue + except asyncio.TimeoutError: + await asyncio.wait_for(self._client.ping(), timeout=Constants.PING_TIMEOUT) + except asyncio.TimeoutError: + self.logger().warning("WebSocket ping timed out. Going to reconnect...") + return + except ConnectionClosed: + return + finally: + await self.disconnect() + + # emit messages + async def _emit(self, method: str, action: str, data: Optional[Dict[str, Any]] = {}) -> int: + payload = { + method: { + "action": action, + **data + } + } + return await self._client.send(json.dumps(payload)) + + # request via websocket + async def request(self, method: str, action: str, data: Optional[Dict[str, Any]] = {}) -> int: + return await self._emit(method, action, data) + + # subscribe to a method + async def subscribe(self, + streams: Optional[Dict[str, Any]] = {}) -> int: + for stream, stream_dict in streams.items(): + if self._isPrivate: + stream_dict = {**stream_dict, **self._auth.get_ws_params()} + await self.request(stream, "subscribe", stream_dict) + return True + + # unsubscribe to a method + async def unsubscribe(self, + streams: Optional[Dict[str, Any]] = {}) -> int: + for stream, stream_dict in streams.items(): + if self._isPrivate: + stream_dict = {**stream_dict, **self._auth.get_ws_params()} + await self.request(stream, "unsubscribe", stream_dict) + return True + + # listen to messages by method + async def on_message(self) -> AsyncIterable[Any]: + async for msg in self._messages(): + yield msg diff --git a/hummingbot/connector/exchange/crypto_com/crypto_com_order_book_tracker.py b/hummingbot/connector/exchange/crypto_com/crypto_com_order_book_tracker.py index 8cf9223b46..cc25e14fc7 100644 --- a/hummingbot/connector/exchange/crypto_com/crypto_com_order_book_tracker.py +++ b/hummingbot/connector/exchange/crypto_com/crypto_com_order_book_tracker.py @@ -83,8 +83,7 @@ async def _track_single_book(self, trading_pair: str): # Output some statistics periodically. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Processed %d order book diffs for %s.", - diff_messages_accepted, trading_pair) + self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") diff_messages_accepted = 0 last_message_timestamp = now elif message.type is OrderBookMessageType.SNAPSHOT: @@ -98,7 +97,7 @@ async def _track_single_book(self, trading_pair: str): d_bids, d_asks = active_order_tracker.convert_diff_message_to_order_book_row(diff_message) order_book.apply_diffs(d_bids, d_asks, diff_message.update_id) - self.logger().debug("Processed order book snapshot for %s.", trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise except Exception: diff --git a/hummingbot/connector/exchange/digifinex/__init__.py b/hummingbot/connector/exchange/digifinex/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hummingbot/connector/exchange/digifinex/digifinex_active_order_tracker.pxd b/hummingbot/connector/exchange/digifinex/digifinex_active_order_tracker.pxd new file mode 100644 index 0000000000..f9fdfcd81e --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_active_order_tracker.pxd @@ -0,0 +1,10 @@ +# distutils: language=c++ +cimport numpy as np + +cdef class DigifinexActiveOrderTracker: + cdef dict _active_bids + cdef dict _active_asks + + cdef tuple c_convert_diff_message_to_np_arrays(self, object message) + cdef tuple c_convert_snapshot_message_to_np_arrays(self, object message) + cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message) diff --git a/hummingbot/connector/exchange/digifinex/digifinex_active_order_tracker.pyx b/hummingbot/connector/exchange/digifinex/digifinex_active_order_tracker.pyx new file mode 100644 index 0000000000..19ff274da7 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_active_order_tracker.pyx @@ -0,0 +1,169 @@ +# distutils: language=c++ +# distutils: sources=hummingbot/core/cpp/OrderBookEntry.cpp + +import logging +import numpy as np + +from decimal import Decimal +from typing import Dict +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.order_book_row import OrderBookRow + +_logger = None +s_empty_diff = np.ndarray(shape=(0, 4), dtype="float64") +DigifinexOrderBookTrackingDictionary = Dict[Decimal, Dict[str, Dict[str, any]]] + +cdef class DigifinexActiveOrderTracker: + def __init__(self, + active_asks: DigifinexOrderBookTrackingDictionary = None, + active_bids: DigifinexOrderBookTrackingDictionary = None): + super().__init__() + self._active_asks = active_asks or {} + self._active_bids = active_bids or {} + + @classmethod + def logger(cls) -> HummingbotLogger: + global _logger + if _logger is None: + _logger = logging.getLogger(__name__) + return _logger + + @property + def active_asks(self) -> DigifinexOrderBookTrackingDictionary: + return self._active_asks + + @property + def active_bids(self) -> DigifinexOrderBookTrackingDictionary: + return self._active_bids + + # TODO: research this more + def volume_for_ask_price(self, price) -> float: + return NotImplementedError + + # TODO: research this more + def volume_for_bid_price(self, price) -> float: + return NotImplementedError + + def get_rates_and_quantities(self, entry) -> tuple: + # price, quantity + return float(entry[0]), float(entry[1]) + + cdef tuple c_convert_diff_message_to_np_arrays(self, object message): + cdef: + dict content = message.content + list bid_entries = [] + list ask_entries = [] + str order_id + str order_side + str price_raw + object price + dict order_dict + double timestamp = message.timestamp + double amount = 0 + + bid_entries = content["bids"] + ask_entries = content["asks"] + + bids = s_empty_diff + asks = s_empty_diff + + if len(bid_entries) > 0: + bids = np.array( + [[timestamp, + float(price), + float(amount), + message.update_id] + for price, amount in [self.get_rates_and_quantities(entry) for entry in bid_entries]], + dtype="float64", + ndmin=2 + ) + + if len(ask_entries) > 0: + asks = np.array( + [[timestamp, + float(price), + float(amount), + message.update_id] + for price, amount in [self.get_rates_and_quantities(entry) for entry in ask_entries]], + dtype="float64", + ndmin=2 + ) + + return bids, asks + + cdef tuple c_convert_snapshot_message_to_np_arrays(self, object message): + cdef: + float price + float amount + str order_id + dict order_dict + + # Refresh all order tracking. + self._active_bids.clear() + self._active_asks.clear() + timestamp = message.timestamp + content = message.content + + for snapshot_orders, active_orders in [(content["bids"], self._active_bids), (content["asks"], self.active_asks)]: + for order in snapshot_orders: + price, amount = self.get_rates_and_quantities(order) + + order_dict = { + "order_id": timestamp, + "amount": amount + } + + if price in active_orders: + active_orders[price][timestamp] = order_dict + else: + active_orders[price] = { + timestamp: order_dict + } + + cdef: + np.ndarray[np.float64_t, ndim=2] bids = np.array( + [[message.timestamp, + price, + sum([order_dict["amount"] + for order_dict in self._active_bids[price].values()]), + message.update_id] + for price in sorted(self._active_bids.keys(), reverse=True)], dtype="float64", ndmin=2) + np.ndarray[np.float64_t, ndim=2] asks = np.array( + [[message.timestamp, + price, + sum([order_dict["amount"] + for order_dict in self.active_asks[price].values()]), + message.update_id] + for price in sorted(self.active_asks.keys(), reverse=True)], dtype="float64", ndmin=2 + ) + + if bids.shape[1] != 4: + bids = bids.reshape((0, 4)) + if asks.shape[1] != 4: + asks = asks.reshape((0, 4)) + + return bids, asks + + cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message): + cdef: + double trade_type_value = 2.0 + + timestamp = message.timestamp + content = message.content + + return np.array( + [timestamp, trade_type_value, float(content["price"]), float(content["size"])], + dtype="float64" + ) + + def convert_diff_message_to_order_book_row(self, message): + np_bids, np_asks = self.c_convert_diff_message_to_np_arrays(message) + bids_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_bids] + asks_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_asks] + return bids_row, asks_row + + def convert_snapshot_message_to_order_book_row(self, message): + np_bids, np_asks = self.c_convert_snapshot_message_to_np_arrays(message) + bids_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_bids] + asks_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_asks] + return bids_row, asks_row diff --git a/hummingbot/connector/exchange/digifinex/digifinex_api_order_book_data_source.py b/hummingbot/connector/exchange/digifinex/digifinex_api_order_book_data_source.py new file mode 100644 index 0000000000..b4162b365b --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_api_order_book_data_source.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python +import asyncio +import logging +import time +import aiohttp +import traceback +import pandas as pd +import hummingbot.connector.exchange.digifinex.digifinex_constants as constants + +from typing import Optional, List, Dict, Any +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_message import OrderBookMessage +from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource +from hummingbot.core.utils.async_utils import safe_gather +from hummingbot.logger import HummingbotLogger +from . import digifinex_utils +from .digifinex_active_order_tracker import DigifinexActiveOrderTracker +from .digifinex_order_book import DigifinexOrderBook +from .digifinex_websocket import DigifinexWebsocket +# from .digifinex_utils import ms_timestamp_to_s + + +class DigifinexAPIOrderBookDataSource(OrderBookTrackerDataSource): + MAX_RETRIES = 20 + MESSAGE_TIMEOUT = 30.0 + SNAPSHOT_TIMEOUT = 10.0 + + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, trading_pairs: List[str] = None): + super().__init__(trading_pairs) + self._trading_pairs: List[str] = trading_pairs + self._snapshot_msg: Dict[str, any] = {} + + @classmethod + async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, float]: + result = {} + async with aiohttp.ClientSession() as client: + resp = await client.get(f"{constants.REST_URL}/ticker") + resp_json = await resp.json() + for t_pair in trading_pairs: + last_trade = [o["last"] for o in resp_json["ticker"] if o["symbol"] == + digifinex_utils.convert_to_exchange_trading_pair(t_pair)] + if last_trade and last_trade[0] is not None: + result[t_pair] = last_trade[0] + return result + + @staticmethod + async def fetch_trading_pairs() -> List[str]: + async with aiohttp.ClientSession() as client: + async with client.get(f"{constants.REST_URL}/ticker", timeout=10) as response: + if response.status == 200: + from hummingbot.connector.exchange.digifinex.digifinex_utils import \ + convert_from_exchange_trading_pair + try: + data: Dict[str, Any] = await response.json() + return [convert_from_exchange_trading_pair(item["symbol"]) for item in data["ticker"]] + except Exception: + pass + # Do nothing if the request fails -- there will be no autocomplete for kucoin trading pairs + return [] + + @staticmethod + async def get_order_book_data(trading_pair: str) -> Dict[str, any]: + """ + Get whole orderbook + """ + async with aiohttp.ClientSession() as client: + orderbook_response = await client.get( + f"{constants.REST_URL}/order_book?limit=150&symbol=" + f"{digifinex_utils.convert_to_exchange_trading_pair(trading_pair)}" + ) + + if orderbook_response.status != 200: + raise IOError( + f"Error fetching OrderBook for {trading_pair} at {constants.EXCHANGE_NAME}. " + f"HTTP status is {orderbook_response.status}." + ) + + orderbook_data: List[Dict[str, Any]] = await safe_gather(orderbook_response.json()) + orderbook_data = orderbook_data[0] + return orderbook_data + + async def get_new_order_book(self, trading_pair: str) -> OrderBook: + snapshot: Dict[str, Any] = await self.get_order_book_data(trading_pair) + snapshot_timestamp: float = time.time() + snapshot_msg: OrderBookMessage = DigifinexOrderBook.snapshot_message_from_exchange( + snapshot, + snapshot_timestamp, + metadata={"trading_pair": trading_pair} + ) + order_book = self.order_book_create_function() + active_order_tracker: DigifinexActiveOrderTracker = DigifinexActiveOrderTracker() + bids, asks = active_order_tracker.convert_snapshot_message_to_order_book_row(snapshot_msg) + order_book.apply_snapshot(bids, asks, snapshot_msg.update_id) + return order_book + + async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for trades using websocket trade channel + """ + while True: + try: + ws = DigifinexWebsocket() + await ws.connect() + + await ws.subscribe("trades", list(map( + lambda pair: f"{digifinex_utils.convert_to_ws_trading_pair(pair)}", + self._trading_pairs + ))) + + async for response in ws.on_message(): + params = response["params"] + symbol = params[2] + for trade in params[1]: + trade_timestamp: int = trade["time"] + trade_msg: OrderBookMessage = DigifinexOrderBook.trade_message_from_exchange( + trade, + trade_timestamp, + metadata={"trading_pair": digifinex_utils.convert_from_ws_trading_pair(symbol)} + ) + output.put_nowait(trade_msg) + + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error.", exc_info=True) + await asyncio.sleep(5.0) + finally: + await ws.disconnect() + + async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for orderbook diffs using websocket book channel + """ + while True: + try: + ws = DigifinexWebsocket() + await ws.connect() + + await ws.subscribe("depth", list(map( + lambda pair: f"{digifinex_utils.convert_to_ws_trading_pair(pair)}", + self._trading_pairs + ))) + + async for response in ws.on_message(): + if response is None or 'params' not in response: + continue + + params = response["params"] + symbol = params[2] + order_book_data = params[1] + timestamp: int = int(time.time()) + + if params[0] is True: + orderbook_msg: OrderBookMessage = DigifinexOrderBook.snapshot_message_from_exchange( + order_book_data, + timestamp, + metadata={"trading_pair": digifinex_utils.convert_from_ws_trading_pair(symbol)} + ) + else: + orderbook_msg: OrderBookMessage = DigifinexOrderBook.diff_message_from_exchange( + order_book_data, + timestamp, + metadata={"trading_pair": digifinex_utils.convert_from_ws_trading_pair(symbol)} + ) + output.put_nowait(orderbook_msg) + + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unexpected error with WebSocket connection.", + exc_info=True, + app_warning_msg="Unexpected error with WebSocket connection. Retrying in 30 seconds. " + "Check network connection." + ) + await asyncio.sleep(30.0) + finally: + await ws.disconnect() + + async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for orderbook snapshots by fetching orderbook + """ + while True: + try: + for trading_pair in self._trading_pairs: + try: + snapshot: Dict[str, any] = await self.get_order_book_data(trading_pair) + snapshot_timestamp: int = snapshot["date"] + snapshot_msg: OrderBookMessage = DigifinexOrderBook.snapshot_message_from_exchange( + snapshot, + snapshot_timestamp, + metadata={"trading_pair": trading_pair} + ) + output.put_nowait(snapshot_msg) + self.logger().debug(f"Saved order book snapshot for {trading_pair}") + # Be careful not to go above API rate limits. + await asyncio.sleep(5.0) + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().network( + f"Unexpected error with WebSocket connection: {e}", + exc_info=True, + app_warning_msg="Unexpected error with WebSocket connection. Retrying in 5 seconds. " + "Check network connection.\n" + + traceback.format_exc() + ) + await asyncio.sleep(5.0) + this_hour: pd.Timestamp = pd.Timestamp.utcnow().replace(minute=0, second=0, microsecond=0) + next_hour: pd.Timestamp = this_hour + pd.Timedelta(hours=1) + delta: float = next_hour.timestamp() - time.time() + await asyncio.sleep(delta) + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error.", exc_info=True) + await asyncio.sleep(5.0) diff --git a/hummingbot/connector/exchange/digifinex/digifinex_api_user_stream_data_source.py b/hummingbot/connector/exchange/digifinex/digifinex_api_user_stream_data_source.py new file mode 100644 index 0000000000..8059190bc5 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_api_user_stream_data_source.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python + +import time +import asyncio +import logging +from typing import Optional, List, AsyncIterable, Any +from hummingbot.connector.exchange.digifinex.digifinex_global import DigifinexGlobal +from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource +from hummingbot.logger import HummingbotLogger +# from .digifinex_auth import DigifinexAuth +from hummingbot.connector.exchange.digifinex.digifinex_websocket import DigifinexWebsocket +from hummingbot.connector.exchange.digifinex import digifinex_utils + + +class DigifinexAPIUserStreamDataSource(UserStreamTrackerDataSource): + MAX_RETRIES = 20 + MESSAGE_TIMEOUT = 30.0 + + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, _global: DigifinexGlobal, trading_pairs: Optional[List[str]] = []): + self._global: DigifinexGlobal = _global + self._trading_pairs = trading_pairs + self._current_listen_key = None + self._listen_for_user_stream_task = None + self._last_recv_time: float = 0 + super().__init__() + + @property + def last_recv_time(self) -> float: + return self._last_recv_time + + async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: + """ + Subscribe to active orders via web socket + """ + + try: + ws = DigifinexWebsocket(self._global.auth) + await ws.connect() + await ws.subscribe("order", list(map( + lambda pair: f"{digifinex_utils.convert_to_ws_trading_pair(pair)}", + self._trading_pairs + ))) + + currencies = set() + for trade_pair in self._trading_pairs: + trade_pair_currencies = trade_pair.split('-') + currencies.update(trade_pair_currencies) + await ws.subscribe("balance", currencies) + + balance_snapshot = await self._global.rest_api.get_balance() + # { + # "code": 0, + # "list": [ + # { + # "currency": "BTC", + # "free": 4723846.89208129, + # "total": 0 + # } + # ] + # } + yield {'method': 'balance.update', 'params': balance_snapshot['list']} + self._last_recv_time = time.time() + + # await ws.subscribe(["user.order", "user.trade", "user.balance"]) + async for msg in ws.on_message(): + # { + # "method": "balance.update", + # "params": [{ + # "currency": "USDT", + # "free": "99944652.8478545303601106", + # "total": "99944652.8478545303601106", + # "used": "0.0000000000" + # }], + # "id": null + # } + yield msg + self._last_recv_time = time.time() + if (msg.get("result") is None): + continue + except Exception as e: + self.logger().exception(e) + raise e + finally: + await ws.disconnect() + await asyncio.sleep(5) + + async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue) -> AsyncIterable[Any]: + """ + *required + Subscribe to user stream via web socket, and keep the connection open for incoming messages + :param ev_loop: ev_loop to execute this function in + :param output: an async queue where the incoming messages are stored + """ + + while True: + try: + async for msg in self._listen_to_orders_trades_balances(): + output.put_nowait(msg) + except asyncio.CancelledError: + raise + except Exception: + self.logger().error( + "Unexpected error with Digifinex WebSocket connection. " "Retrying after 30 seconds...", exc_info=True + ) + await asyncio.sleep(30.0) diff --git a/hummingbot/connector/exchange/digifinex/digifinex_auth.py b/hummingbot/connector/exchange/digifinex/digifinex_auth.py new file mode 100644 index 0000000000..d90e2da161 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_auth.py @@ -0,0 +1,77 @@ +import hmac +import hashlib +import base64 +import urllib +import aiohttp +from typing import List, Dict, Any +# from hummingbot.connector.exchange.digifinex.digifinex_utils import get_ms_timestamp +from hummingbot.connector.exchange.digifinex import digifinex_constants as Constants +from hummingbot.connector.exchange.digifinex.time_patcher import TimePatcher +# import time + +_time_patcher: TimePatcher = None + + +def time_patcher() -> TimePatcher: + global _time_patcher + if _time_patcher is None: + _time_patcher = TimePatcher('Digifinex', DigifinexAuth.query_time_func) + _time_patcher.start() + return _time_patcher + + +class DigifinexAuth(): + """ + Auth class required by digifinex API + Learn more at https://docs.digifinex.io/en-ww/v3/#signature-authentication-amp-verification + """ + def __init__(self, api_key: str, secret_key: str): + self.api_key = api_key + self.secret_key = secret_key + self.time_patcher = time_patcher() + # self.time_patcher = time + + @classmethod + async def query_time_func() -> float: + async with aiohttp.ClientSession() as session: + async with session.get(Constants.REST_URL + '/time') as resp: + resp_data: Dict[str, float] = await resp.json() + return float(resp_data["server_time"]) + + def get_private_headers( + self, + path_url: str, + request_id: int, + data: Dict[str, Any] = None + ): + + data = data or {} + payload = urllib.parse.urlencode(data) + sig = hmac.new( + self.secret_key.encode('utf-8'), + payload.encode('utf-8'), + hashlib.sha256 + ).hexdigest() + nonce = int(self.time_patcher.time()) + + header = { + 'ACCESS-KEY': self.api_key, + 'ACCESS-TIMESTAMP': str(nonce), + 'ACCESS-SIGN': sig, + } + + return header + + def generate_ws_signature(self) -> List[Any]: + data = [None] * 3 + data[0] = self.api_key + nonce = int(self.time_patcher.time() * 1000) + data[1] = str(nonce) + + data[2] = base64.b64encode(hmac.new( + self.secret_key.encode('latin-1'), + f"{nonce}".encode('latin-1'), + hashlib.sha256 + ).digest()) + + return data diff --git a/hummingbot/connector/exchange/digifinex/digifinex_constants.py b/hummingbot/connector/exchange/digifinex/digifinex_constants.py new file mode 100644 index 0000000000..0caa476ba7 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_constants.py @@ -0,0 +1,39 @@ +# A single source of truth for constant variables related to the exchange +import os + +if os.environ.get('digifinex_test') == '1': + host = 'openapi.digifinex.vip' +else: + host = 'openapi.digifinex.com' + +EXCHANGE_NAME = "digifinex" +REST_URL = f"https://{host}/v3" +WSS_PRIVATE_URL = f"wss://{host}/ws/v1/" +WSS_PUBLIC_URL = f"wss://{host}/ws/v1/" + +API_REASONS = { + 0: "Success", + 10001: "Malformed request, (E.g. not using application/json for REST)", + 10002: "Not authenticated, or key/signature incorrect", + 10003: "IP address not whitelisted", + 10004: "Missing required fields", + 10005: "Disallowed based on user tier", + 10006: "Requests have exceeded rate limits", + 10007: "Nonce value differs by more than 30 seconds from server", + 10008: "Invalid method specified", + 10009: "Invalid date range", + 20001: "Duplicated record", + 20002: "Insufficient balance", + 30003: "Invalid instrument_name specified", + 30004: "Invalid side specified", + 30005: "Invalid type specified", + 30006: "Price is lower than the minimum", + 30007: "Price is higher than the maximum", + 30008: "Quantity is lower than the minimum", + 30009: "Quantity is higher than the maximum", + 30010: "Required argument is blank or missing", + 30013: "Too many decimal places for Price", + 30014: "Too many decimal places for Quantity", + 30016: "The notional amount is less than the minimum", + 30017: "The notional amount exceeds the maximum", +} diff --git a/hummingbot/connector/exchange/digifinex/digifinex_exchange.py b/hummingbot/connector/exchange/digifinex/digifinex_exchange.py new file mode 100644 index 0000000000..db01913616 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_exchange.py @@ -0,0 +1,829 @@ +import logging +from typing import ( + Dict, + List, + Optional, + Any, + AsyncIterable, +) +from decimal import Decimal +import asyncio +# import json +# import aiohttp +import math +import time + +from hummingbot.core.network_iterator import NetworkStatus +from hummingbot.logger import HummingbotLogger +from hummingbot.core.clock import Clock +from hummingbot.core.utils import estimate_fee +from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather +from hummingbot.connector.trading_rule import TradingRule +from hummingbot.core.data_type.cancellation_result import CancellationResult +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.limit_order import LimitOrder +from hummingbot.core.event.events import ( + MarketEvent, + BuyOrderCompletedEvent, + SellOrderCompletedEvent, + OrderFilledEvent, + OrderCancelledEvent, + BuyOrderCreatedEvent, + SellOrderCreatedEvent, + MarketOrderFailureEvent, + OrderType, + TradeType, + TradeFee +) +from hummingbot.connector.exchange_base import ExchangeBase +from hummingbot.connector.exchange.digifinex.digifinex_global import DigifinexGlobal +from hummingbot.connector.exchange.digifinex.digifinex_order_book_tracker import DigifinexOrderBookTracker +from hummingbot.connector.exchange.digifinex.digifinex_user_stream_tracker import DigifinexUserStreamTracker +# from hummingbot.connector.exchange.digifinex.digifinex_auth import DigifinexAuth +from hummingbot.connector.exchange.digifinex.digifinex_in_flight_order import DigifinexInFlightOrder +from hummingbot.connector.exchange.digifinex import digifinex_utils +# from hummingbot.connector.exchange.digifinex import digifinex_constants as Constants +from hummingbot.core.data_type.common import OpenOrder +ctce_logger = None +s_decimal_NaN = Decimal("nan") + + +class DigifinexExchange(ExchangeBase): + """ + DigifinexExchange connects with digifinex.com exchange and provides order book pricing, user account tracking and + trading functionality. + """ + API_CALL_TIMEOUT = 10.0 + SHORT_POLL_INTERVAL = 5.0 + UPDATE_ORDER_STATUS_MIN_INTERVAL = 10.0 + LONG_POLL_INTERVAL = 120.0 + + @classmethod + def logger(cls) -> HummingbotLogger: + global ctce_logger + if ctce_logger is None: + ctce_logger = logging.getLogger(__name__) + return ctce_logger + + def __init__(self, + digifinex_api_key: str, + digifinex_secret_key: str, + trading_pairs: Optional[List[str]] = None, + trading_required: bool = True + ): + """ + :param key: The API key to connect to private digifinex.com APIs. + :param secret: The API secret. + :param trading_pairs: The market trading pairs which to track order book data. + :param trading_required: Whether actual trading is needed. + """ + super().__init__() + self._trading_required = trading_required + self._trading_pairs = trading_pairs + self._global = DigifinexGlobal(digifinex_api_key, digifinex_secret_key) + # self._rest_api = DigifinexRestApi(self._digifinex_auth, self._http_client) + self._order_book_tracker = DigifinexOrderBookTracker(trading_pairs=trading_pairs) + self._user_stream_tracker = DigifinexUserStreamTracker(self._global, trading_pairs) + self._ev_loop = asyncio.get_event_loop() + self._poll_notifier = asyncio.Event() + self._last_timestamp = 0 + self._in_flight_orders: Dict[str, DigifinexInFlightOrder] = {} # Dict[client_order_id:str, DigifinexInFlightOrder] + self._order_not_found_records = {} # Dict[client_order_id:str, count:int] + self._trading_rules = {} # Dict[trading_pair:str, TradingRule] + self._status_polling_task = None + self._user_stream_event_listener_task = None + self._trading_rules_polling_task = None + self._last_poll_timestamp = 0 + + @property + def name(self) -> str: + return "digifinex" + + @property + def order_books(self) -> Dict[str, OrderBook]: + return self._order_book_tracker.order_books + + @property + def trading_rules(self) -> Dict[str, TradingRule]: + return self._trading_rules + + @property + def in_flight_orders(self) -> Dict[str, DigifinexInFlightOrder]: + return self._in_flight_orders + + @property + def status_dict(self) -> Dict[str, bool]: + """ + A dictionary of statuses of various connector's components. + """ + return { + "order_books_initialized": self._order_book_tracker.ready, + "account_balance": len(self._account_balances) > 0 if self._trading_required else True, + "trading_rule_initialized": len(self._trading_rules) > 0, + "user_stream_initialized": + self._user_stream_tracker.data_source.last_recv_time > 0 if self._trading_required else True, + } + + @property + def ready(self) -> bool: + """ + :return True when all statuses pass, this might take 5-10 seconds for all the connector's components and + services to be ready. + """ + return all(self.status_dict.values()) + + @property + def limit_orders(self) -> List[LimitOrder]: + return [ + in_flight_order.to_limit_order() + for in_flight_order in self._in_flight_orders.values() + ] + + @property + def tracking_states(self) -> Dict[str, any]: + """ + :return active in-flight orders in json format, is used to save in sqlite db. + """ + return { + key: value.to_json() + for key, value in self._in_flight_orders.items() + if not value.is_done + } + + def restore_tracking_states(self, saved_states: Dict[str, any]): + """ + Restore in-flight orders from saved tracking states, this is st the connector can pick up on where it left off + when it disconnects. + :param saved_states: The saved tracking_states. + """ + self._in_flight_orders.update({ + key: DigifinexInFlightOrder.from_json(value) + for key, value in saved_states.items() + }) + + def supported_order_types(self) -> List[OrderType]: + """ + :return a list of OrderType supported by this connector. + Note that Market order type is no longer required and will not be used. + """ + return [OrderType.LIMIT, OrderType.LIMIT_MAKER] + + def start(self, clock: Clock, timestamp: float): + """ + This function is called automatically by the clock. + """ + super().start(clock, timestamp) + + def stop(self, clock: Clock): + """ + This function is called automatically by the clock. + """ + super().stop(clock) + + async def start_network(self): + """ + This function is required by NetworkIterator base class and is called automatically. + It starts tracking order book, polling trading rules, + updating statuses and tracking user data. + """ + self._order_book_tracker.start() + self._trading_rules_polling_task = safe_ensure_future(self._trading_rules_polling_loop()) + if self._trading_required: + self._status_polling_task = safe_ensure_future(self._status_polling_loop()) + self._user_stream_tracker_task = safe_ensure_future(self._user_stream_tracker.start()) + self._user_stream_event_listener_task = safe_ensure_future(self._user_stream_event_listener()) + + async def stop_network(self): + """ + This function is required by NetworkIterator base class and is called automatically. + """ + self._order_book_tracker.stop() + if self._status_polling_task is not None: + self._status_polling_task.cancel() + self._status_polling_task = None + if self._trading_rules_polling_task is not None: + self._trading_rules_polling_task.cancel() + self._trading_rules_polling_task = None + if self._status_polling_task is not None: + self._status_polling_task.cancel() + self._status_polling_task = None + if self._user_stream_tracker_task is not None: + self._user_stream_tracker_task.cancel() + self._user_stream_tracker_task = None + if self._user_stream_event_listener_task is not None: + self._user_stream_event_listener_task.cancel() + self._user_stream_event_listener_task = None + + async def check_network(self) -> NetworkStatus: + """ + This function is required by NetworkIterator base class and is called periodically to check + the network connection. Simply ping the network (or call any light weight public API). + """ + try: + await self._global.rest_api.request("get", "ping") + except asyncio.CancelledError: + raise + except Exception as e: + _ = e + self.logger().exception('check_network', stack_info=True) + return NetworkStatus.NOT_CONNECTED + return NetworkStatus.CONNECTED + + async def _trading_rules_polling_loop(self): + """ + Periodically update trading rule. + """ + while True: + try: + await self._update_trading_rules() + await asyncio.sleep(60) + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().network(f"Unexpected error while fetching trading rules. Error: {str(e)}", + exc_info=True, + app_warning_msg="Could not fetch new trading rules from digifinex.com. " + "Check network connection.") + await asyncio.sleep(0.5) + + async def _update_trading_rules(self): + instruments_info = await self._global.rest_api.request("get", path_url="markets") + self._trading_rules.clear() + self._trading_rules = self._format_trading_rules(instruments_info) + + def _format_trading_rules(self, instruments_info: Dict[str, Any]) -> Dict[str, TradingRule]: + """ + Converts json API response into a dictionary of trading rules. + :param instruments_info: The json API response + :return A dictionary of trading rules. + Response Example: + { + "data": [{ + "volume_precision": 4, + "price_precision": 2, + "market": "btc_usdt", + "min_amount": 2, + "min_volume": 0.0001 + }], + "date": 1589873858, + "code": 0 + } + """ + result = {} + for rule in instruments_info["data"]: + try: + trading_pair = digifinex_utils.convert_from_exchange_trading_pair(rule["market"]) + price_decimals = Decimal(str(rule["price_precision"])) + quantity_decimals = Decimal(str(rule["volume_precision"])) + # E.g. a price decimal of 2 means 0.01 incremental. + price_step = Decimal("1") / Decimal(str(math.pow(10, price_decimals))) + quantity_step = Decimal("1") / Decimal(str(math.pow(10, quantity_decimals))) + result[trading_pair] = TradingRule(trading_pair, + min_price_increment=price_step, + min_base_amount_increment=quantity_step) + except Exception: + self.logger().error(f"Error parsing the trading pair rule {rule}. Skipping.", exc_info=True) + return result + + def get_order_price_quantum(self, trading_pair: str, price: Decimal): + """ + Returns a price step, a minimum price increment for a given trading pair. + """ + trading_rule = self._trading_rules[trading_pair] + return trading_rule.min_price_increment + + def get_order_size_quantum(self, trading_pair: str, order_size: Decimal): + """ + Returns an order amount step, a minimum amount increment for a given trading pair. + """ + trading_rule = self._trading_rules[trading_pair] + return Decimal(trading_rule.min_base_amount_increment) + + def get_order_book(self, trading_pair: str) -> OrderBook: + if trading_pair not in self._order_book_tracker.order_books: + raise ValueError(f"No order book exists for '{trading_pair}'.") + return self._order_book_tracker.order_books[trading_pair] + + def buy(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, + price: Decimal = s_decimal_NaN, **kwargs) -> str: + """ + Buys an amount of base asset (of the given trading pair). This function returns immediately. + To see an actual order, you'll have to wait for BuyOrderCreatedEvent. + :param trading_pair: The market (e.g. BTC-USDT) to buy from + :param amount: The amount in base token value + :param order_type: The order type + :param price: The price (note: this is no longer optional) + :returns A new internal order id + """ + order_id: str = digifinex_utils.get_new_client_order_id(True, trading_pair) + safe_ensure_future(self._create_order(TradeType.BUY, order_id, trading_pair, amount, order_type, price)) + return order_id + + def sell(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, + price: Decimal = s_decimal_NaN, **kwargs) -> str: + """ + Sells an amount of base asset (of the given trading pair). This function returns immediately. + To see an actual order, you'll have to wait for SellOrderCreatedEvent. + :param trading_pair: The market (e.g. BTC-USDT) to sell from + :param amount: The amount in base token value + :param order_type: The order type + :param price: The price (note: this is no longer optional) + :returns A new internal order id + """ + order_id: str = digifinex_utils.get_new_client_order_id(False, trading_pair) + safe_ensure_future(self._create_order(TradeType.SELL, order_id, trading_pair, amount, order_type, price)) + return order_id + + def cancel(self, trading_pair: str, order_id: str): + """ + Cancel an order. This function returns immediately. + To get the cancellation result, you'll have to wait for OrderCancelledEvent. + :param trading_pair: The market (e.g. BTC-USDT) of the order. + :param order_id: The internal order id (also called client_order_id) + """ + tracked_order = self._in_flight_orders.get(order_id) + if tracked_order is None: + raise ValueError(f"Failed to cancel order - {order_id}. Order not found.") + if tracked_order.exchange_order_id is None: + self.ev_loop.run_until_complete(tracked_order.get_exchange_order_id()) + safe_ensure_future(self._execute_cancel(tracked_order)) + return order_id + + async def _create_order(self, + trade_type: TradeType, + order_id: str, + trading_pair: str, + amount: Decimal, + order_type: OrderType, + price: Decimal): + """ + Calls create-order API end point to place an order, starts tracking the order and triggers order created event. + :param trade_type: BUY or SELL + :param order_id: Internal order id (also called client_order_id) + :param trading_pair: The market to place order + :param amount: The order amount (in base token value) + :param order_type: The order type + :param price: The order price + """ + if not order_type.is_limit_type(): + raise Exception(f"Unsupported order type: {order_type}") + trading_rule = self._trading_rules[trading_pair] + + amount = self.quantize_order_amount(trading_pair, amount) + price = self.quantize_order_price(trading_pair, price) + if amount < trading_rule.min_order_size: + raise ValueError(f"Buy order amount {amount} is lower than the minimum order size " + f"{trading_rule.min_order_size}.") + symbol = digifinex_utils.convert_to_exchange_trading_pair(trading_pair) + api_params = {"symbol": symbol, + "type": trade_type.name.lower(), + "price": f"{price:f}", + "amount": f"{amount:f}", + # "client_oid": order_id + } + if order_type is OrderType.LIMIT_MAKER: + api_params["post_only"] = 1 + self.start_tracking_order(order_id, + None, + trading_pair, + trade_type, + price, + amount, + order_type + ) + try: + order_result = await self._global.rest_api.request("post", "spot/order/new", api_params, True) + exchange_order_id = str(order_result["order_id"]) + tracked_order = self._in_flight_orders.get(order_id) + if tracked_order is not None: + self.logger().info(f"Created {order_type.name} {trade_type.name} order {order_id} for " + f"{amount} {trading_pair}.") + tracked_order.update_exchange_order_id(exchange_order_id) + + event_tag = MarketEvent.BuyOrderCreated if trade_type is TradeType.BUY else MarketEvent.SellOrderCreated + event_class = BuyOrderCreatedEvent if trade_type is TradeType.BUY else SellOrderCreatedEvent + self.trigger_event(event_tag, + event_class( + self.current_timestamp, + order_type, + trading_pair, + amount, + price, + order_id + )) + except asyncio.CancelledError: + raise + except Exception as e: + self.stop_tracking_order(order_id) + self.logger().network( + f"Error submitting {trade_type.name} {order_type.name} order to Digifinex for " + f"{amount} {trading_pair} " + f"{price}.", + exc_info=True, + app_warning_msg=str(e) + ) + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent(self.current_timestamp, order_id, order_type)) + + def start_tracking_order(self, + order_id: str, + exchange_order_id: str, + trading_pair: str, + trade_type: TradeType, + price: Decimal, + amount: Decimal, + order_type: OrderType): + """ + Starts tracking an order by simply adding it into _in_flight_orders dictionary. + """ + self._in_flight_orders[order_id] = DigifinexInFlightOrder( + client_order_id=order_id, + exchange_order_id=exchange_order_id, + trading_pair=trading_pair, + order_type=order_type, + trade_type=trade_type, + price=price, + amount=amount + ) + + def stop_tracking_order(self, order_id: str): + """ + Stops tracking an order by simply removing it from _in_flight_orders dictionary. + """ + if order_id in self._in_flight_orders: + del self._in_flight_orders[order_id] + + async def _execute_cancel(self, o: DigifinexInFlightOrder) -> str: + """ + Executes order cancellation process by first calling cancel-order API. The API result doesn't confirm whether + the cancellation is successful, it simply states it receives the request. + :param trading_pair: The market trading pair + :param order_id: The internal order id + order.last_state to change to CANCELED + """ + try: + await self._global.rest_api.request( + "post", + "spot/order/cancel", + {"order_id": o.exchange_order_id}, + True + ) + if o.client_order_id in self._in_flight_orders: + self.trigger_event(MarketEvent.OrderCancelled, + OrderCancelledEvent(self.current_timestamp, o.client_order_id)) + del self._in_flight_orders[o.client_order_id] + return o.exchange_order_id + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().network( + f"Failed to cancel order {o.exchange_order_id}: {str(e)}", + exc_info=True, + app_warning_msg=f"Failed to cancel the order {o.exchange_order_id} on Digifinex. " + f"Check API key and network connection." + ) + + async def _status_polling_loop(self): + """ + Periodically update user balances and order status via REST API. This serves as a fallback measure for web + socket API updates. + """ + while True: + try: + self._poll_notifier = asyncio.Event() + await self._poll_notifier.wait() + await safe_gather( + self._update_balances(), + self._update_order_status(), + ) + self._last_poll_timestamp = self.current_timestamp + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().error(str(e), exc_info=True) + self.logger().network("Unexpected error while fetching account updates.", + exc_info=True, + app_warning_msg="Could not fetch account updates from Digifinex. " + "Check API key and network connection.") + await asyncio.sleep(0.5) + + async def _update_balances(self): + local_asset_names = set(self._account_balances.keys()) + remote_asset_names = set() + account_info = await self._global.rest_api.get_balance() + for account in account_info["list"]: + asset_name = account["currency"] + self._account_available_balances[asset_name] = Decimal(str(account["free"])) + self._account_balances[asset_name] = Decimal(str(account["total"])) + remote_asset_names.add(asset_name) + + try: + asset_names_to_remove = local_asset_names.difference(remote_asset_names) + for asset_name in asset_names_to_remove: + del self._account_available_balances[asset_name] + del self._account_balances[asset_name] + except Exception as e: + self.logger().error(e) + + async def _update_order_status(self): + """ + Calls REST API to get status update for each in-flight order. + """ + last_tick = int(self._last_poll_timestamp / self.UPDATE_ORDER_STATUS_MIN_INTERVAL) + current_tick = int(self.current_timestamp / self.UPDATE_ORDER_STATUS_MIN_INTERVAL) + + if current_tick > last_tick and len(self._in_flight_orders) > 0: + tracked_orders = list(self._in_flight_orders.values()) + tasks = [] + for tracked_order in tracked_orders: + order_id = await tracked_order.get_exchange_order_id() + tasks.append(self._global.rest_api.request("get", + "spot/order/detail", + {"order_id": order_id}, + True)) + self.logger().debug(f"Polling for order status updates of {len(tasks)} orders.") + update_results = await safe_gather(*tasks, return_exceptions=True) + for update_result in update_results: + if isinstance(update_result, Exception): + raise update_result + if "data" not in update_result: + self.logger().info(f"_update_order_status result not in resp: {update_result}") + continue + order_data = update_result["data"] + self._process_rest_trade_details(order_data) + self._process_order_status(order_data.get('order_id'), order_data.get('status')) + + def _process_order_status(self, exchange_order_id: str, status: int): + """ + Updates in-flight order and triggers cancellation or failure event if needed. + """ + tracked_order = self.find_exchange_order(exchange_order_id) + if tracked_order is None: + return + client_order_id = tracked_order.client_order_id + # Update order execution status + tracked_order.last_state = str(status) + if tracked_order.is_cancelled: + self.logger().info(f"Successfully cancelled order {client_order_id}.") + self.trigger_event(MarketEvent.OrderCancelled, + OrderCancelledEvent( + self.current_timestamp, + client_order_id)) + tracked_order.cancelled_event.set() + self.stop_tracking_order(client_order_id) + # elif tracked_order.is_failure: + # self.logger().info(f"The market order {client_order_id} has failed according to order status API. " + # f"Reason: {digifinex_utils.get_api_reason(order_msg['reason'])}") + # self.trigger_event(MarketEvent.OrderFailure, + # MarketOrderFailureEvent( + # self.current_timestamp, + # client_order_id, + # tracked_order.order_type + # )) + # self.stop_tracking_order(client_order_id) + + def _process_rest_trade_details(self, order_detail_msg: Any): + for trade_msg in order_detail_msg['detail']: + """ + Updates in-flight order and trigger order filled event for trade message received. Triggers order completed + event if the total executed amount equals to the specified order amount. + """ + # for order in self._in_flight_orders.values(): + # await order.get_exchange_order_id() + tracked_order = self.find_exchange_order(trade_msg['order_id']) + if tracked_order is None: + return + + updated = tracked_order.update_with_rest_order_detail(trade_msg) + if not updated: + return + + self.trigger_event( + MarketEvent.OrderFilled, + OrderFilledEvent( + self.current_timestamp, + tracked_order.client_order_id, + tracked_order.trading_pair, + tracked_order.trade_type, + tracked_order.order_type, + Decimal(str(trade_msg["executed_price"])), + Decimal(str(trade_msg["executed_amount"])), + estimate_fee.estimate_fee(self.name, tracked_order.order_type in [OrderType.LIMIT, OrderType.LIMIT_MAKER]), + # TradeFee(0.0, [(trade_msg["fee_currency"], Decimal(str(trade_msg["fee"])))]), + exchange_trade_id=trade_msg["tid"] + ) + ) + if math.isclose(tracked_order.executed_amount_base, tracked_order.amount) or \ + tracked_order.executed_amount_base >= tracked_order.amount: + tracked_order.last_state = "FILLED" + self.logger().info(f"The {tracked_order.trade_type.name} order " + f"{tracked_order.client_order_id} has completed " + f"according to order status API.") + event_tag = MarketEvent.BuyOrderCompleted if tracked_order.trade_type is TradeType.BUY \ + else MarketEvent.SellOrderCompleted + event_class = BuyOrderCompletedEvent if tracked_order.trade_type is TradeType.BUY \ + else SellOrderCompletedEvent + self.trigger_event(event_tag, + event_class(self.current_timestamp, + tracked_order.client_order_id, + tracked_order.base_asset, + tracked_order.quote_asset, + tracked_order.fee_asset, + tracked_order.executed_amount_base, + tracked_order.executed_amount_quote, + tracked_order.fee_paid, + tracked_order.order_type)) + self.stop_tracking_order(tracked_order.client_order_id) + + def find_exchange_order(self, exchange_order_id: str): + for o in self._in_flight_orders.values(): + if o.exchange_order_id == exchange_order_id: + return o + + def _process_order_message_traded(self, order_msg): + tracked_order: DigifinexInFlightOrder = self.find_exchange_order(order_msg['id']) + if tracked_order is None: + return + + (delta_trade_amount, delta_trade_price) = tracked_order.update_with_order_update(order_msg) + if not delta_trade_amount: + return + + self.trigger_event( + MarketEvent.OrderFilled, + OrderFilledEvent( + self.current_timestamp, + tracked_order.client_order_id, + tracked_order.trading_pair, + tracked_order.trade_type, + tracked_order.order_type, + delta_trade_price, + delta_trade_amount, + estimate_fee.estimate_fee(self.name, tracked_order.order_type in [OrderType.LIMIT, OrderType.LIMIT_MAKER]), + # TradeFee(0.0, [(trade_msg["fee_currency"], Decimal(str(trade_msg["fee"])))]), + exchange_trade_id='N/A' + ) + ) + if math.isclose(tracked_order.executed_amount_base, tracked_order.amount) or \ + tracked_order.executed_amount_base >= tracked_order.amount: + tracked_order.last_state = "2" + self.logger().info(f"The {tracked_order.trade_type.name} order " + f"{tracked_order.client_order_id} has completed " + f"according to order status API.") + event_tag = MarketEvent.BuyOrderCompleted if tracked_order.trade_type is TradeType.BUY \ + else MarketEvent.SellOrderCompleted + event_class = BuyOrderCompletedEvent if tracked_order.trade_type is TradeType.BUY \ + else SellOrderCompletedEvent + self.trigger_event(event_tag, + event_class(self.current_timestamp, + tracked_order.client_order_id, + tracked_order.base_asset, + tracked_order.quote_asset, + tracked_order.fee_asset, + tracked_order.executed_amount_base, + tracked_order.executed_amount_quote, + tracked_order.fee_paid, + tracked_order.order_type)) + self.stop_tracking_order(tracked_order.client_order_id) + + async def cancel_all(self, timeout_seconds: float): + """ + Cancels all in-flight orders and waits for cancellation results. + Used by bot's top level stop and exit commands (cancelling outstanding orders on exit) + :param timeout_seconds: The timeout at which the operation will be canceled. + :returns List of CancellationResult which indicates whether each order is successfully cancelled. + """ + if self._trading_pairs is None: + raise Exception("cancel_all can only be used when trading_pairs are specified.") + cancellation_results = [] + try: + # for trading_pair in self._trading_pairs: + # await self._global.rest_api.request( + # "post", + # "private/cancel-all-orders", + # {"instrument_name": digifinex_utils.convert_to_exchange_trading_pair(trading_pair)}, + # True + # ) + + open_orders = list(self._in_flight_orders.values()) + for o in open_orders: + await self._execute_cancel(o) + + for cl_order_id, tracked_order in self._in_flight_orders.items(): + open_order = [o for o in open_orders if o.exchange_order_id == tracked_order.exchange_order_id] + if not open_order: + cancellation_results.append(CancellationResult(cl_order_id, True)) + # self.trigger_event(MarketEvent.OrderCancelled, + # OrderCancelledEvent(self.current_timestamp, cl_order_id)) + else: + cancellation_results.append(CancellationResult(cl_order_id, False)) + except Exception: + self.logger().network( + "Failed to cancel all orders.", + exc_info=True, + app_warning_msg="Failed to cancel all orders on Digifinex. Check API key and network connection." + ) + return cancellation_results + + def tick(self, timestamp: float): + """ + Is called automatically by the clock for each clock's tick (1 second by default). + It checks if status polling task is due for execution. + """ + now = time.time() + poll_interval = (self.SHORT_POLL_INTERVAL + if now - self._user_stream_tracker.last_recv_time > 60.0 + else self.LONG_POLL_INTERVAL) + last_tick = int(self._last_timestamp / poll_interval) + current_tick = int(timestamp / poll_interval) + if current_tick > last_tick: + if not self._poll_notifier.is_set(): + self._poll_notifier.set() + self._last_timestamp = timestamp + + def get_fee(self, + base_currency: str, + quote_currency: str, + order_type: OrderType, + order_side: TradeType, + amount: Decimal, + price: Decimal = s_decimal_NaN) -> TradeFee: + """ + To get trading fee, this function is simplified by using fee override configuration. Most parameters to this + function are ignore except order_type. Use OrderType.LIMIT_MAKER to specify you want trading fee for + maker order. + """ + is_maker = order_type is OrderType.LIMIT_MAKER + return TradeFee(percent=self.estimate_fee_pct(is_maker)) + + async def _iter_user_event_queue(self) -> AsyncIterable[Dict[str, any]]: + while True: + try: + yield await self._user_stream_tracker.user_stream.get() + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unknown error. Retrying after 1 seconds.", + exc_info=True, + app_warning_msg="Could not fetch user events from Digifinex. Check API key and network connection." + ) + await asyncio.sleep(1.0) + + async def _user_stream_event_listener(self): + """ + Listens to message in _user_stream_tracker.user_stream queue. The messages are put in by + DigifinexAPIUserStreamDataSource. + """ + async for event_message in self._iter_user_event_queue(): + try: + if "method" not in event_message: + continue + channel = event_message["method"] + # if "user.trade" in channel: + # for trade_msg in event_message["result"]["data"]: + # await self._process_trade_message(trade_msg) + if "order.update" in channel: + for order_msg in event_message["params"]: + self._process_order_status(order_msg['id'], order_msg['status']) + self._process_order_message_traded(order_msg) + elif channel == "balance.update": + balances = event_message["params"] + for balance_entry in balances: + asset_name = balance_entry["currency"] + self._account_balances[asset_name] = Decimal(str(balance_entry["total"])) + self._account_available_balances[asset_name] = Decimal(str(balance_entry["free"])) + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error in user stream listener loop.", exc_info=True) + await asyncio.sleep(5.0) + + async def get_open_orders(self) -> List[OpenOrder]: + result = await self._global.rest_api.request( + "get", + "spot/order/current", + {}, + True + ) + ret_val = [] + for order in result["data"]: + # if digifinex_utils.HBOT_BROKER_ID not in order["client_oid"]: + # continue + if order["type"] not in ["buy", "sell"]: + raise Exception(f"Unsupported order type {order['type']}") + ret_val.append( + OpenOrder( + client_order_id=None, + trading_pair=digifinex_utils.convert_from_exchange_trading_pair(order["symbol"]), + price=Decimal(str(order["price"])), + amount=Decimal(str(order["amount"])), + executed_amount=Decimal(str(order["executed_amount"])), + status=order["status"], + order_type=OrderType.LIMIT, + is_buy=True if order["type"] == "buy" else False, + time=int(order["created_date"]), + exchange_order_id=order["order_id"] + ) + ) + return ret_val diff --git a/hummingbot/connector/exchange/digifinex/digifinex_global.py b/hummingbot/connector/exchange/digifinex/digifinex_global.py new file mode 100644 index 0000000000..13b8350d96 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_global.py @@ -0,0 +1,19 @@ +import aiohttp +from hummingbot.connector.exchange.digifinex.digifinex_auth import DigifinexAuth +from hummingbot.connector.exchange.digifinex.digifinex_rest_api import DigifinexRestApi + + +class DigifinexGlobal: + + def __init__(self, key: str, secret: str): + self.auth = DigifinexAuth(key, secret) + self.rest_api = DigifinexRestApi(self.auth, self.http_client) + self._shared_client: aiohttp.ClientSession = None + + async def http_client(self) -> aiohttp.ClientSession: + """ + :returns Shared client session instance + """ + if self._shared_client is None: + self._shared_client = aiohttp.ClientSession() + return self._shared_client diff --git a/hummingbot/connector/exchange/digifinex/digifinex_in_flight_order.py b/hummingbot/connector/exchange/digifinex/digifinex_in_flight_order.py new file mode 100644 index 0000000000..4c7382acee --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_in_flight_order.py @@ -0,0 +1,123 @@ +from decimal import Decimal +from typing import ( + Any, + Dict, + Optional, + Tuple, +) +import asyncio +from hummingbot.core.event.events import ( + OrderType, + TradeType +) +from hummingbot.connector.in_flight_order_base import InFlightOrderBase + + +class DigifinexInFlightOrder(InFlightOrderBase): + def __init__(self, + client_order_id: str, + exchange_order_id: Optional[str], + trading_pair: str, + order_type: OrderType, + trade_type: TradeType, + price: Decimal, + amount: Decimal, + initial_state: str = "OPEN"): + super().__init__( + client_order_id, + exchange_order_id, + trading_pair, + order_type, + trade_type, + price, + amount, + initial_state, + ) + self.trade_id_set = set() + self.cancelled_event = asyncio.Event() + + @property + def is_done(self) -> bool: + return self.last_state in {"2", "3", "4"} + + @property + def is_failure(self) -> bool: + return False + # return self.last_state in {"REJECTED"} + + @property + def is_cancelled(self) -> bool: + return self.last_state in {"3", "4"} + + # @property + # def order_type_description(self) -> str: + # """ + # :return: Order description string . One of ["limit buy" / "limit sell" / "market buy" / "market sell"] + # """ + # order_type = "market" if self.order_type is OrderType.MARKET else "limit" + # side = "buy" if self.trade_type == TradeType.BUY else "sell" + # return f"{order_type} {side}" + + @classmethod + def from_json(cls, data: Dict[str, Any]) -> InFlightOrderBase: + """ + :param data: json data from API + :return: formatted InFlightOrder + """ + retval = DigifinexInFlightOrder( + data["client_order_id"], + data["exchange_order_id"], + data["trading_pair"], + getattr(OrderType, data["order_type"]), + getattr(TradeType, data["trade_type"]), + Decimal(data["price"]), + Decimal(data["amount"]), + data["last_state"] + ) + retval.executed_amount_base = Decimal(data["executed_amount_base"]) + retval.executed_amount_quote = Decimal(data["executed_amount_quote"]) + retval.fee_asset = data["fee_asset"] + retval.fee_paid = Decimal(data["fee_paid"]) + retval.last_state = data["last_state"] + return retval + + def update_with_rest_order_detail(self, trade_update: Dict[str, Any]) -> bool: + """ + Updates the in flight order with trade update (from private/get-order-detail end point) + return: True if the order gets updated otherwise False + """ + trade_id = trade_update["tid"] + # trade_update["orderId"] is type int + if trade_id in self.trade_id_set: + # trade already recorded + return False + self.trade_id_set.add(trade_id) + self.executed_amount_base += Decimal(str(trade_update["executed_amount"])) + # self.fee_paid += Decimal(str(trade_update["fee"])) + self.executed_amount_quote += (Decimal(str(trade_update["executed_price"])) * + Decimal(str(trade_update["executed_amount"]))) + # if not self.fee_asset: + # self.fee_asset = trade_update["fee_currency"] + return True + + def update_with_order_update(self, order_update) -> Tuple[Decimal, Decimal]: + """ + Updates the in flight order with trade update (from order message) + return: (delta_trade_amount, delta_trade_price) + """ + + # todo: order_msg contains no trade_id. may be re-processed + if order_update['filled'] == '0': + return (0, 0) + + self.trade_id_set.add("N/A") + executed_amount_base = Decimal(order_update['filled']) + if executed_amount_base == self.executed_amount_base: + return (0, 0) + delta_trade_amount = executed_amount_base - self.executed_amount_base + self.executed_amount_base = executed_amount_base + + executed_amount_quote = executed_amount_base * Decimal(order_update['price_avg'] or order_update['price']) + delta_trade_price = (executed_amount_quote - self.executed_amount_quote) / delta_trade_amount + self.executed_amount_quote = executed_amount_base * Decimal(order_update['price_avg'] or order_update['price']) + return (delta_trade_amount, delta_trade_price) diff --git a/hummingbot/connector/exchange/digifinex/digifinex_order_book.py b/hummingbot/connector/exchange/digifinex/digifinex_order_book.py new file mode 100644 index 0000000000..29ecae9727 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_order_book.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python + +import logging +import hummingbot.connector.exchange.digifinex.digifinex_constants as constants + +from sqlalchemy.engine import RowProxy +from typing import ( + Optional, + Dict, + List, Any) +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_message import ( + OrderBookMessage, OrderBookMessageType +) +from hummingbot.connector.exchange.digifinex.digifinex_order_book_message import DigifinexOrderBookMessage + +_logger = None + + +class DigifinexOrderBook(OrderBook): + @classmethod + def logger(cls) -> HummingbotLogger: + global _logger + if _logger is None: + _logger = logging.getLogger(__name__) + return _logger + + @classmethod + def snapshot_message_from_exchange(cls, + msg: Dict[str, any], + timestamp: float, + metadata: Optional[Dict] = None): + """ + Convert json snapshot data into standard OrderBookMessage format + :param msg: json snapshot data from live web socket stream + :param timestamp: timestamp attached to incoming data + :return: DigifinexOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + return DigifinexOrderBookMessage( + message_type=OrderBookMessageType.SNAPSHOT, + content=msg, + timestamp=timestamp + ) + + @classmethod + def snapshot_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of snapshot data into standard OrderBookMessage format + :param record: a row of snapshot data from the database + :return: DigifinexOrderBookMessage + """ + return DigifinexOrderBookMessage( + message_type=OrderBookMessageType.SNAPSHOT, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def diff_message_from_exchange(cls, + msg: Dict[str, any], + timestamp: Optional[float] = None, + metadata: Optional[Dict] = None): + """ + Convert json diff data into standard OrderBookMessage format + :param msg: json diff data from live web socket stream + :param timestamp: timestamp attached to incoming data + :return: DigifinexOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + return DigifinexOrderBookMessage( + message_type=OrderBookMessageType.DIFF, + content=msg, + timestamp=timestamp + ) + + @classmethod + def diff_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of diff data into standard OrderBookMessage format + :param record: a row of diff data from the database + :return: DigifinexOrderBookMessage + """ + return DigifinexOrderBookMessage( + message_type=OrderBookMessageType.DIFF, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def trade_message_from_exchange(cls, + msg: Dict[str, Any], + timestamp: Optional[float] = None, + metadata: Optional[Dict] = None): + """ + Convert a trade data into standard OrderBookMessage format + :param record: a trade data from the database + :return: DigifinexOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + msg.update({ + "exchange_order_id": msg.get("id"), + "trade_type": msg.get("type"), + "price": msg.get("price"), + "amount": msg.get("amount"), + }) + + return DigifinexOrderBookMessage( + message_type=OrderBookMessageType.TRADE, + content=msg, + timestamp=timestamp + ) + + @classmethod + def trade_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of trade data into standard OrderBookMessage format + :param record: a row of trade data from the database + :return: DigifinexOrderBookMessage + """ + return DigifinexOrderBookMessage( + message_type=OrderBookMessageType.TRADE, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def from_snapshot(cls, snapshot: OrderBookMessage): + raise NotImplementedError(constants.EXCHANGE_NAME + " order book needs to retain individual order data.") + + @classmethod + def restore_from_snapshot_and_diffs(self, snapshot: OrderBookMessage, diffs: List[OrderBookMessage]): + raise NotImplementedError(constants.EXCHANGE_NAME + " order book needs to retain individual order data.") diff --git a/hummingbot/connector/exchange/digifinex/digifinex_order_book_message.py b/hummingbot/connector/exchange/digifinex/digifinex_order_book_message.py new file mode 100644 index 0000000000..883ea99da7 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_order_book_message.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python + +from typing import ( + Dict, + List, + Optional, +) + +from hummingbot.core.data_type.order_book_row import OrderBookRow +from hummingbot.core.data_type.order_book_message import ( + OrderBookMessage, + OrderBookMessageType, +) + + +class DigifinexOrderBookMessage(OrderBookMessage): + def __new__( + cls, + message_type: OrderBookMessageType, + content: Dict[str, any], + timestamp: Optional[float] = None, + *args, + **kwargs, + ): + if timestamp is None: + if message_type is OrderBookMessageType.SNAPSHOT: + raise ValueError("timestamp must not be None when initializing snapshot messages.") + timestamp = content["timestamp"] + + return super(DigifinexOrderBookMessage, cls).__new__( + cls, message_type, content, timestamp=timestamp, *args, **kwargs + ) + + @property + def update_id(self) -> int: + if self.type in [OrderBookMessageType.DIFF, OrderBookMessageType.SNAPSHOT]: + return int(self.timestamp * 1e3) + else: + return -1 + + @property + def trade_id(self) -> int: + if self.type is OrderBookMessageType.TRADE: + return int(self.timestamp * 1e3) + return -1 + + @property + def trading_pair(self) -> str: + if "trading_pair" in self.content: + return self.content["trading_pair"] + elif "instrument_name" in self.content: + return self.content["instrument_name"] + + @property + def asks(self) -> List[OrderBookRow]: + asks = map(self.content["asks"], lambda ask: {"price": ask[0], "amount": ask[1]}) + + return [ + OrderBookRow(float(price), float(amount), self.update_id) for price, amount in asks + ] + + @property + def bids(self) -> List[OrderBookRow]: + bids = map(self.content["bids"], lambda bid: {"price": bid[0], "amount": bid[1]}) + + return [ + OrderBookRow(float(price), float(amount), self.update_id) for price, amount in bids + ] + + def __eq__(self, other) -> bool: + return self.type == other.type and self.timestamp == other.timestamp + + def __lt__(self, other) -> bool: + if self.timestamp != other.timestamp: + return self.timestamp < other.timestamp + else: + """ + If timestamp is the same, the ordering is snapshot < diff < trade + """ + return self.type.value < other.type.value diff --git a/hummingbot/connector/exchange/bitmax/bitmax_order_book_tracker.py b/hummingbot/connector/exchange/digifinex/digifinex_order_book_tracker.py similarity index 75% rename from hummingbot/connector/exchange/bitmax/bitmax_order_book_tracker.py rename to hummingbot/connector/exchange/digifinex/digifinex_order_book_tracker.py index 9fa26cc508..a90d5fb035 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_order_book_tracker.py +++ b/hummingbot/connector/exchange/digifinex/digifinex_order_book_tracker.py @@ -2,20 +2,21 @@ import asyncio import bisect import logging -import hummingbot.connector.exchange.bitmax.bitmax_constants as constants +import hummingbot.connector.exchange.digifinex.digifinex_constants as constants import time + from collections import defaultdict, deque from typing import Optional, Dict, List, Deque from hummingbot.core.data_type.order_book_message import OrderBookMessageType from hummingbot.logger import HummingbotLogger from hummingbot.core.data_type.order_book_tracker import OrderBookTracker -from hummingbot.connector.exchange.bitmax.bitmax_order_book_message import BitmaxOrderBookMessage -from hummingbot.connector.exchange.bitmax.bitmax_active_order_tracker import BitmaxActiveOrderTracker -from hummingbot.connector.exchange.bitmax.bitmax_api_order_book_data_source import BitmaxAPIOrderBookDataSource -from hummingbot.connector.exchange.bitmax.bitmax_order_book import BitmaxOrderBook +from hummingbot.connector.exchange.digifinex.digifinex_order_book_message import DigifinexOrderBookMessage +from hummingbot.connector.exchange.digifinex.digifinex_active_order_tracker import DigifinexActiveOrderTracker +from hummingbot.connector.exchange.digifinex.digifinex_api_order_book_data_source import DigifinexAPIOrderBookDataSource +from hummingbot.connector.exchange.digifinex.digifinex_order_book import DigifinexOrderBook -class BitmaxOrderBookTracker(OrderBookTracker): +class DigifinexOrderBookTracker(OrderBookTracker): _logger: Optional[HummingbotLogger] = None @classmethod @@ -25,7 +26,7 @@ def logger(cls) -> HummingbotLogger: return cls._logger def __init__(self, trading_pairs: Optional[List[str]] = None,): - super().__init__(BitmaxAPIOrderBookDataSource(trading_pairs), trading_pairs) + super().__init__(DigifinexAPIOrderBookDataSource(trading_pairs), trading_pairs) self._ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() self._order_book_snapshot_stream: asyncio.Queue = asyncio.Queue() @@ -33,10 +34,10 @@ def __init__(self, trading_pairs: Optional[List[str]] = None,): self._order_book_trade_stream: asyncio.Queue = asyncio.Queue() self._process_msg_deque_task: Optional[asyncio.Task] = None self._past_diffs_windows: Dict[str, Deque] = {} - self._order_books: Dict[str, BitmaxOrderBook] = {} - self._saved_message_queues: Dict[str, Deque[BitmaxOrderBookMessage]] = \ + self._order_books: Dict[str, DigifinexOrderBook] = {} + self._saved_message_queues: Dict[str, Deque[DigifinexOrderBookMessage]] = \ defaultdict(lambda: deque(maxlen=1000)) - self._active_order_trackers: Dict[str, BitmaxActiveOrderTracker] = defaultdict(BitmaxActiveOrderTracker) + self._active_order_trackers: Dict[str, DigifinexActiveOrderTracker] = defaultdict(DigifinexActiveOrderTracker) self._order_book_stream_listener_task: Optional[asyncio.Task] = None self._order_book_trade_listener_task: Optional[asyncio.Task] = None @@ -51,20 +52,20 @@ async def _track_single_book(self, trading_pair: str): """ Update an order book with changes from the latest batch of received messages """ - past_diffs_window: Deque[BitmaxOrderBookMessage] = deque() + past_diffs_window: Deque[DigifinexOrderBookMessage] = deque() self._past_diffs_windows[trading_pair] = past_diffs_window message_queue: asyncio.Queue = self._tracking_message_queues[trading_pair] - order_book: BitmaxOrderBook = self._order_books[trading_pair] - active_order_tracker: BitmaxActiveOrderTracker = self._active_order_trackers[trading_pair] + order_book: DigifinexOrderBook = self._order_books[trading_pair] + active_order_tracker: DigifinexActiveOrderTracker = self._active_order_trackers[trading_pair] last_message_timestamp: float = time.time() diff_messages_accepted: int = 0 while True: try: - message: BitmaxOrderBookMessage = None - saved_messages: Deque[BitmaxOrderBookMessage] = self._saved_message_queues[trading_pair] + message: DigifinexOrderBookMessage = None + saved_messages: Deque[DigifinexOrderBookMessage] = self._saved_message_queues[trading_pair] # Process saved messages first if there are any if len(saved_messages) > 0: message = saved_messages.popleft() @@ -87,7 +88,7 @@ async def _track_single_book(self, trading_pair: str): diff_messages_accepted = 0 last_message_timestamp = now elif message.type is OrderBookMessageType.SNAPSHOT: - past_diffs: List[BitmaxOrderBookMessage] = list(past_diffs_window) + past_diffs: List[DigifinexOrderBookMessage] = list(past_diffs_window) # only replay diffs later than snapshot, first update active order with snapshot then replay diffs replay_position = bisect.bisect_right(past_diffs, message) replay_diffs = past_diffs[replay_position:] diff --git a/hummingbot/connector/exchange/digifinex/digifinex_order_book_tracker_entry.py b/hummingbot/connector/exchange/digifinex/digifinex_order_book_tracker_entry.py new file mode 100644 index 0000000000..afc487dc70 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_order_book_tracker_entry.py @@ -0,0 +1,21 @@ +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_tracker_entry import OrderBookTrackerEntry +from hummingbot.connector.exchange.digifinex.digifinex_active_order_tracker import DigifinexActiveOrderTracker + + +class DigifinexOrderBookTrackerEntry(OrderBookTrackerEntry): + def __init__( + self, trading_pair: str, timestamp: float, order_book: OrderBook, active_order_tracker: DigifinexActiveOrderTracker + ): + self._active_order_tracker = active_order_tracker + super(DigifinexOrderBookTrackerEntry, self).__init__(trading_pair, timestamp, order_book) + + def __repr__(self) -> str: + return ( + f"DigifinexOrderBookTrackerEntry(trading_pair='{self._trading_pair}', timestamp='{self._timestamp}', " + f"order_book='{self._order_book}')" + ) + + @property + def active_order_tracker(self) -> DigifinexActiveOrderTracker: + return self._active_order_tracker diff --git a/hummingbot/connector/exchange/digifinex/digifinex_rest_api.py b/hummingbot/connector/exchange/digifinex/digifinex_rest_api.py new file mode 100644 index 0000000000..34ca77afb5 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_rest_api.py @@ -0,0 +1,112 @@ +from typing import Callable, Dict, Any +import aiohttp +import json +import urllib +from hummingbot.connector.exchange.digifinex.digifinex_auth import DigifinexAuth +from hummingbot.connector.exchange.digifinex import digifinex_constants as Constants +from hummingbot.connector.exchange.digifinex import digifinex_utils + + +class DigifinexRestApi: + + def __init__(self, auth: DigifinexAuth, http_client_getter: Callable[[], aiohttp.ClientSession]): + self._auth = auth + self._http_client = http_client_getter + + async def request(self, + method: str, + path_url: str, + params: Dict[str, Any] = {}, + is_auth_required: bool = False) -> Dict[str, Any]: + """ + Sends an aiohttp request and waits for a response. + :param method: The HTTP method, e.g. get or post + :param path_url: The path url or the API end point + :param is_auth_required: Whether an authentication is required, when True the function will add encrypted + signature to the request. + :returns A response in json format. + """ + url = f"{Constants.REST_URL}/{path_url}" + client = await self._http_client() + if is_auth_required: + request_id = digifinex_utils.RequestId.generate_request_id() + headers = self._auth.get_private_headers(path_url, request_id, params) + else: + headers = {} + headers['User-Agent'] = 'hummingbot' + + if method == "get": + url = f'{url}?{urllib.parse.urlencode(params)}' + response = await client.get(url, headers=headers) + elif method == "post": + response = await client.post(url, data=params, headers=headers) + else: + raise NotImplementedError + + try: + parsed_response = json.loads(await response.text()) + except Exception as e: + raise IOError(f"Error parsing data from {url}. Error: {str(e)}") + if response.status != 200: + raise IOError(f"Error fetching data from {url}. HTTP status is {response.status}. " + f"Message: {parsed_response}") + code = parsed_response["code"] + if code != 0: + msgs = { + 10001: "Wrong request method, please check it's a GET or POST request", + 10002: "Invalid ApiKey", + 10003: "Sign doesn't match", + 10004: "Illegal request parameters", + 10005: "Request frequency exceeds the limit", + 10006: "Unauthorized to execute this request", + 10007: "IP address Unauthorized", + 10008: "Timestamp for this request is invalid", + 10009: "Unexist endpoint or misses ACCESS-KEY, please check endpoint URL", + 10011: "ApiKey expired. Please go to client side to re-create an ApiKey.", + 20002: "Trade of this trading pair is suspended", + 20007: "Price precision error", + 20008: "Amount precision error", + 20009: "Amount is less than the minimum requirement", + 20010: "Cash Amount is less than the minimum requirement", + 20011: "Insufficient balance", + 20012: "Invalid trade type (valid value: buy/sell)", + 20013: "No order info found", + 20014: "Invalid date (Valid format: 2018-07-25)", + 20015: "Date exceeds the limit", + 20018: "Your have been banned for API trading by the system", + 20019: 'Wrong trading pair symbol, correct format:"base_quote", e.g. "btc_usdt"', + 20020: "You have violated the API trading rules and temporarily banned for trading. At present, we have certain restrictions on the user's transaction rate and withdrawal rate.", + 20021: "Invalid currency", + 20022: "The ending timestamp must be larger than the starting timestamp", + 20023: "Invalid transfer type", + 20024: "Invalid amount", + 20025: "This currency is not transferable at the moment", + 20026: "Transfer amount exceed your balance", + 20027: "Abnormal account status", + 20028: "Blacklist for transfer", + 20029: "Transfer amount exceed your daily limit", + 20030: "You have no position on this trading pair", + 20032: "Withdrawal limited", + 20033: "Wrong Withdrawal ID", + 20034: "Withdrawal service of this crypto has been closed", + 20035: "Withdrawal limit", + 20036: "Withdrawal cancellation failed", + 20037: "The withdrawal address, Tag or chain type is not included in the withdrawal management list", + 20038: "The withdrawal address is not on the white list", + 20039: "Can't be canceled in current status", + 20040: "Withdraw too frequently; limitation: 3 times a minute, 100 times a day", + 20041: "Beyond the daily withdrawal limit", + 20042: "Current trading pair does not support API trading", + 50000: "Exception error", + } + raise IOError(f"{url} API call failed, response: {parsed_response} ({msgs[code]})") + # print(f"REQUEST: {method} {path_url} {params}") + # print(f"RESPONSE: {parsed_response}") + return parsed_response + + async def get_balance(self) -> Dict[str, Any]: + """ + Calls REST API to update total and available balances. + """ + account_info = await self.request("get", "spot/assets", {}, True) + return account_info diff --git a/hummingbot/connector/exchange/bitmax/bitmax_user_stream_tracker.py b/hummingbot/connector/exchange/digifinex/digifinex_user_stream_tracker.py similarity index 78% rename from hummingbot/connector/exchange/bitmax/bitmax_user_stream_tracker.py rename to hummingbot/connector/exchange/digifinex/digifinex_user_stream_tracker.py index 8255abdb94..fe0125a6a3 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_user_stream_tracker.py +++ b/hummingbot/connector/exchange/digifinex/digifinex_user_stream_tracker.py @@ -15,13 +15,13 @@ safe_ensure_future, safe_gather, ) -from hummingbot.connector.exchange.bitmax.bitmax_api_user_stream_data_source import \ - BitmaxAPIUserStreamDataSource -from hummingbot.connector.exchange.bitmax.bitmax_auth import BitmaxAuth -from hummingbot.connector.exchange.bitmax.bitmax_constants import EXCHANGE_NAME +from hummingbot.connector.exchange.digifinex.digifinex_api_user_stream_data_source import \ + DigifinexAPIUserStreamDataSource +from hummingbot.connector.exchange.digifinex.digifinex_global import DigifinexGlobal +from hummingbot.connector.exchange.digifinex.digifinex_constants import EXCHANGE_NAME -class BitmaxUserStreamTracker(UserStreamTracker): +class DigifinexUserStreamTracker(UserStreamTracker): _cbpust_logger: Optional[HummingbotLogger] = None @classmethod @@ -31,10 +31,10 @@ def logger(cls) -> HummingbotLogger: return cls._bust_logger def __init__(self, - bitmax_auth: Optional[BitmaxAuth] = None, + _global: DigifinexGlobal, trading_pairs: Optional[List[str]] = []): super().__init__() - self._bitmax_auth: BitmaxAuth = bitmax_auth + self._global: DigifinexGlobal = _global self._trading_pairs: List[str] = trading_pairs self._ev_loop: asyncio.events.AbstractEventLoop = asyncio.get_event_loop() self._data_source: Optional[UserStreamTrackerDataSource] = None @@ -48,8 +48,8 @@ def data_source(self) -> UserStreamTrackerDataSource: :return: OrderBookTrackerDataSource """ if not self._data_source: - self._data_source = BitmaxAPIUserStreamDataSource( - bitmax_auth=self._bitmax_auth, + self._data_source = DigifinexAPIUserStreamDataSource( + self._global, trading_pairs=self._trading_pairs ) return self._data_source diff --git a/hummingbot/connector/exchange/digifinex/digifinex_utils.py b/hummingbot/connector/exchange/digifinex/digifinex_utils.py new file mode 100644 index 0000000000..a48ac5d3b1 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_utils.py @@ -0,0 +1,98 @@ +import math +from typing import Dict, List + +from hummingbot.core.utils.tracking_nonce import get_tracking_nonce, get_tracking_nonce_low_res +from . import digifinex_constants as Constants + +from hummingbot.client.config.config_var import ConfigVar +from hummingbot.client.config.config_methods import using_exchange + + +CENTRALIZED = True + +EXAMPLE_PAIR = "ETH-USDT" + +DEFAULT_FEES = [0.1, 0.1] + +HBOT_BROKER_ID = "HBOT-" + + +# deeply merge two dictionaries +def merge_dicts(source: Dict, destination: Dict) -> Dict: + for key, value in source.items(): + if isinstance(value, dict): + # get node or create one + node = destination.setdefault(key, {}) + merge_dicts(value, node) + else: + destination[key] = value + + return destination + + +# join paths +def join_paths(*paths: List[str]) -> str: + return "/".join(paths) + + +# get timestamp in milliseconds +def get_ms_timestamp() -> int: + return get_tracking_nonce_low_res() + + +# convert milliseconds timestamp to seconds +def ms_timestamp_to_s(ms: int) -> int: + return math.floor(ms / 1e3) + + +# Request ID class +class RequestId: + """ + Generate request ids + """ + _request_id: int = 0 + + @classmethod + def generate_request_id(cls) -> int: + return get_tracking_nonce() + + +def convert_from_exchange_trading_pair(exchange_trading_pair: str) -> str: + return exchange_trading_pair.replace("_", "-").upper() + + +def convert_from_ws_trading_pair(exchange_trading_pair: str) -> str: + return exchange_trading_pair.replace("_", "-") + + +def convert_to_exchange_trading_pair(hb_trading_pair: str) -> str: + return hb_trading_pair.replace("-", "_").lower() + + +def convert_to_ws_trading_pair(hb_trading_pair: str) -> str: + return hb_trading_pair.replace("-", "_") + + +def get_new_client_order_id(is_buy: bool, trading_pair: str) -> str: + side = "B" if is_buy else "S" + return f"{HBOT_BROKER_ID}{side}-{trading_pair}-{get_tracking_nonce()}" + + +def get_api_reason(code: str) -> str: + return Constants.API_REASONS.get(int(code), code) + + +KEYS = { + "digifinex_api_key": + ConfigVar(key="digifinex_api_key", + prompt="Enter your Digifinex API key >>> ", + required_if=using_exchange("digifinex"), + is_secure=True, + is_connect_key=True), + "digifinex_secret_key": + ConfigVar(key="digifinex_secret_key", + prompt="Enter your Digifinex secret key >>> ", + required_if=using_exchange("digifinex"), + is_secure=True, + is_connect_key=True), +} diff --git a/hummingbot/connector/exchange/digifinex/digifinex_websocket.py b/hummingbot/connector/exchange/digifinex/digifinex_websocket.py new file mode 100644 index 0000000000..c2ca969778 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_websocket.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python +import asyncio +import copy +import logging +import websockets +import zlib +import ujson +from asyncio import InvalidStateError +import hummingbot.connector.exchange.digifinex.digifinex_constants as constants +# from hummingbot.core.utils.async_utils import safe_ensure_future + + +from typing import Optional, AsyncIterable, Any, List +from websockets.exceptions import ConnectionClosed +from hummingbot.logger import HummingbotLogger +from hummingbot.connector.exchange.digifinex.digifinex_auth import DigifinexAuth +from hummingbot.connector.exchange.digifinex.digifinex_utils import RequestId + +# reusable websocket class +# ToDo: We should eventually remove this class, and instantiate web socket connection normally (see Binance for example) + + +class DigifinexWebsocket(RequestId): + MESSAGE_TIMEOUT = 30.0 + PING_TIMEOUT = 10.0 + _logger: Optional[HummingbotLogger] = None + disconnect_future: asyncio.Future = None + tasks: [asyncio.Task] = [] + login_msg_id: int = 0 + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, auth: Optional[DigifinexAuth] = None): + self._auth: Optional[DigifinexAuth] = auth + self._isPrivate = True if self._auth is not None else False + self._WS_URL = constants.WSS_PRIVATE_URL if self._isPrivate else constants.WSS_PUBLIC_URL + self._client: Optional[websockets.WebSocketClientProtocol] = None + + # connect to exchange + async def connect(self): + if self.disconnect_future is not None: + raise InvalidStateError('already connected') + self.disconnect_future = asyncio.Future() + + try: + self._client = await websockets.connect(self._WS_URL) + + # if auth class was passed into websocket class + # we need to emit authenticated requests + if self._isPrivate: + await self.login() + self.tasks.append(asyncio.create_task(self._ping_loop())) + + return self._client + except Exception as e: + self.logger().error(f"Websocket error: '{str(e)}'", exc_info=True) + + async def login(self): + self.login_msg_id = await self._emit("server.auth", self._auth.generate_ws_signature()) + msg = await self._messages() + if msg is None: + raise ConnectionError('websocket auth failed: connection closed unexpectedly') + if msg.get('error') is not None: + raise ConnectionError(f'websocket auth failed: {msg}') + + # disconnect from exchange + async def disconnect(self): + if self._client is None: + return + + await self._client.close() + if not self.disconnect_future.done: + self.disconnect_future.result(True) + if len(self.tasks) > 0: + await asyncio.wait(self.tasks) + + async def _ping_loop(self): + while True: + try: + disconnected = await asyncio.wait_for(self.disconnect_future, 30) + _ = disconnected + break + except asyncio.TimeoutError: + await self._emit('server.ping', []) + # msg = await self._messages() # concurrent read not allowed + + # receive & parse messages + async def _messages(self) -> Any: + try: + success = False + while True: + try: + raw_msg_bytes: bytes = await asyncio.wait_for(self._client.recv(), timeout=self.MESSAGE_TIMEOUT) + inflated_msg: bytes = zlib.decompress(raw_msg_bytes) + raw_msg = ujson.loads(inflated_msg) + # if "method" in raw_msg and raw_msg["method"] == "server.ping": + # payload = {"id": raw_msg["id"], "method": "public/respond-heartbeat"} + # safe_ensure_future(self._client.send(ujson.dumps(payload))) + # self.logger().debug(inflated_msg) + # method = raw_msg.get('method') + # if method not in ['depth.update', 'trades.update']: + # self.logger().network(inflated_msg) + + err = raw_msg.get('error') + if err is not None: + raise ConnectionError(raw_msg) + elif raw_msg.get('result') == 'pong': + continue # ignore ping response + + success = True + return raw_msg + except asyncio.TimeoutError: + await asyncio.wait_for(self._client.ping(), timeout=self.PING_TIMEOUT) + except asyncio.TimeoutError: + self.logger().warning("WebSocket ping timed out. Going to reconnect...") + return + except ConnectionClosed: + return + except Exception as e: + _ = e + self.logger().exception('digifinex.websocket._messages', stack_info=True) + raise + finally: + if not success: + await self.disconnect() + + # emit messages + async def _emit(self, method: str, data: Optional[Any] = {}) -> int: + id = self.generate_request_id() + + payload = { + "id": id, + "method": method, + "params": copy.deepcopy(data), + } + + req = ujson.dumps(payload) + self.logger().network(req) # todo remove log + await self._client.send(req) + + return id + + # request via websocket + async def request(self, method: str, data: Optional[Any] = {}) -> int: + return await self._emit(method, data) + + # subscribe to a method + async def subscribe(self, category: str, channels: List[str]) -> int: + id = await self.request(category + ".subscribe", channels) + msg = await self._messages() + if msg.get('error') is not None: + raise ConnectionError(f'subscribe {category} {channels} failed: {msg}') + return id + + # unsubscribe to a method + async def unsubscribe(self, channels: List[str]) -> int: + return await self.request("unsubscribe", { + "channels": channels + }) + + # listen to messages by method + async def on_message(self) -> AsyncIterable[Any]: + while True: + msg = await self._messages() + if msg is None: + return + if 'pong' in str(msg): + _ = int(0) + yield msg diff --git a/hummingbot/connector/exchange/digifinex/time_patcher.py b/hummingbot/connector/exchange/digifinex/time_patcher.py new file mode 100644 index 0000000000..3c6d195f30 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/time_patcher.py @@ -0,0 +1,112 @@ +import asyncio +from collections import deque +import logging +import statistics +import time +from typing import Deque, Optional, Callable, Awaitable + +from hummingbot.logger import HummingbotLogger +from hummingbot.core.utils.async_utils import safe_ensure_future + + +class TimePatcher: + # BINANCE_TIME_API = "https://api.binance.com/api/v1/time" + NaN = float("nan") + _bt_logger = None + _bt_shared_instance = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._bt_logger is None: + cls._bt_logger = logging.getLogger(__name__) + return cls._bt_logger + + # query_time_func returns the server time in seconds + def __init__(self, exchange_name: str, query_time_func: Callable[[], Awaitable[float]], check_interval: float = 60.0): + self._exchange_name = exchange_name + self._query_time_func = query_time_func + self._time_offset_ms: Deque[float] = deque([]) + self._set_server_time_offset_task: Optional[asyncio.Task] = None + self._started: bool = False + self._server_time_offset_check_interval = check_interval + self._median_window = 5 + self._last_update_local_time: float = self.NaN + self._scheduled_update_task: Optional[asyncio.Task] = None + + @property + def started(self) -> bool: + return self._started + + @property + def time_offset_ms(self) -> float: + if not self._time_offset_ms: + return (time.time() - time.perf_counter()) * 1e3 + return statistics.median(self._time_offset_ms) + + def add_time_offset_ms_sample(self, offset: float): + self._time_offset_ms.append(offset) + while len(self._time_offset_ms) > self._median_window: + self._time_offset_ms.popleft() + + def clear_time_offset_ms_samples(self): + self._time_offset_ms.clear() + + def time(self) -> float: + return time.perf_counter() + self.time_offset_ms * 1e-3 + + def start(self): + if self._set_server_time_offset_task is None: + self._set_server_time_offset_task = safe_ensure_future(self.update_server_time_offset_loop()) + self._started = True + + def stop(self): + if self._set_server_time_offset_task: + self._set_server_time_offset_task.cancel() + self._set_server_time_offset_task = None + self._time_offset_ms.clear() + self._started = False + + def schedule_update_server_time_offset(self) -> asyncio.Task: + # If an update task is already scheduled, don't do anything. + if self._scheduled_update_task is not None and not self._scheduled_update_task.done(): + return self._scheduled_update_task + + current_local_time: float = time.perf_counter() + if not (current_local_time - self._last_update_local_time < 5): + # If there was no recent update, schedule the server time offset update immediately. + self._scheduled_update_task = safe_ensure_future(self.update_server_time_offset()) + else: + # If there was a recent update, schedule the server time offset update after 5 seconds. + async def update_later(): + await asyncio.sleep(5.0) + await self.update_server_time_offset() + self._scheduled_update_task = safe_ensure_future(update_later()) + + return self._scheduled_update_task + + async def update_server_time_offset_loop(self): + while True: + await self.update_server_time_offset() + await asyncio.sleep(self._server_time_offset_check_interval) + + async def update_server_time_offset(self): + try: + local_before_ms: float = time.perf_counter() * 1e3 + query_time_func = self._query_time_func.__func__ + server_time = await query_time_func() + # async with aiohttp.ClientSession() as session: + # async with session.get(self.BINANCE_TIME_API) as resp: + # resp_data: Dict[str, float] = await resp.json() + # binance_server_time_ms: float = float(resp_data["serverTime"]) + # local_after_ms: float = time.perf_counter() * 1e3 + local_after_ms: float = time.perf_counter() * 1e3 + local_server_time_pre_image_ms: float = (local_before_ms + local_after_ms) / 2.0 + time_offset_ms: float = server_time * 1000 - local_server_time_pre_image_ms + self.add_time_offset_ms_sample(time_offset_ms) + self._last_update_local_time = time.perf_counter() + except asyncio.CancelledError: + raise + except Exception: + self.logger().network(f"Error getting {self._exchange_name} server time.", exc_info=True, + app_warning_msg=f"Could not refresh {self._exchange_name} server time. " + "Check network connection.") diff --git a/hummingbot/connector/exchange/dolomite/dolomite_order_book_tracker.py b/hummingbot/connector/exchange/dolomite/dolomite_order_book_tracker.py index 5c2e50b388..d460d67804 100644 --- a/hummingbot/connector/exchange/dolomite/dolomite_order_book_tracker.py +++ b/hummingbot/connector/exchange/dolomite/dolomite_order_book_tracker.py @@ -107,7 +107,7 @@ async def _track_single_book(self, trading_pair: str): elif message.type is OrderBookMessageType.SNAPSHOT: s_bids, s_asks = active_order_tracker.convert_snapshot_message_to_order_book_row(message) order_book.apply_snapshot(s_bids, s_asks, message.update_id) - self.logger().debug("Processed order book snapshot for %s.", trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise diff --git a/hummingbot/connector/exchange/dydx/dydx_order_book_tracker.py b/hummingbot/connector/exchange/dydx/dydx_order_book_tracker.py index 1450c57156..67763d305e 100644 --- a/hummingbot/connector/exchange/dydx/dydx_order_book_tracker.py +++ b/hummingbot/connector/exchange/dydx/dydx_order_book_tracker.py @@ -86,7 +86,7 @@ async def _track_single_book(self, trading_pair: str): elif message.type is OrderBookMessageType.SNAPSHOT: s_bids, s_asks = active_order_tracker.convert_snapshot_message_to_order_book_row(message) order_book.apply_snapshot(s_bids, s_asks, int(message.timestamp)) - self.logger().debug("Processed order book snapshot for %s.", trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index 9f6f83ec15..ba0ecadcb0 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -42,6 +42,7 @@ from hummingbot.connector.exchange.hitbtc.hitbtc_utils import ( convert_from_exchange_trading_pair, convert_to_exchange_trading_pair, + translate_asset, get_new_client_order_id, aiohttp_response_with_errors, retry_sleep_time, @@ -740,7 +741,7 @@ def _process_balance_message(self, balance_update): local_asset_names = set(self._account_balances.keys()) remote_asset_names = set() for account in balance_update: - asset_name = account["currency"] + asset_name = translate_asset(account["currency"]) self._account_available_balances[asset_name] = Decimal(str(account["available"])) self._account_balances[asset_name] = Decimal(str(account["reserved"])) + Decimal(str(account["available"])) remote_asset_names.add(asset_name) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py index c549ce8b72..3f430227b0 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py @@ -57,18 +57,36 @@ def split_trading_pair(trading_pair: str) -> Optional[Tuple[str, str]]: return None +def translate_asset(asset_name: str) -> str: + asset_replacements = [ + ("USD", "USDT"), + ] + for asset_replacement in asset_replacements: + for inv in [0, 1]: + if asset_name == asset_replacement[inv]: + return asset_replacement[(0 if inv else 1)] + return asset_name + + +def translate_assets(hb_trading_pair: str) -> str: + assets = hb_trading_pair.split('-') + for x in range(len(assets)): + assets[x] = translate_asset(assets[x]) + return '-'.join(assets) + + def convert_from_exchange_trading_pair(ex_trading_pair: str) -> Optional[str]: regex_match = split_trading_pair(ex_trading_pair) if regex_match is None: return None # HitBTC uses uppercase (BTCUSDT) base_asset, quote_asset = split_trading_pair(ex_trading_pair) - return f"{base_asset.upper()}-{quote_asset.upper()}" + return translate_assets(f"{base_asset.upper()}-{quote_asset.upper()}") def convert_to_exchange_trading_pair(hb_trading_pair: str) -> str: # HitBTC uses uppercase (BTCUSDT) - return hb_trading_pair.replace("-", "").upper() + return translate_assets(hb_trading_pair).replace("-", "").upper() def get_new_client_order_id(is_buy: bool, trading_pair: str) -> str: diff --git a/hummingbot/connector/exchange/huobi/huobi_order_book_tracker.py b/hummingbot/connector/exchange/huobi/huobi_order_book_tracker.py index 39c83f429b..f66d0e0053 100644 --- a/hummingbot/connector/exchange/huobi/huobi_order_book_tracker.py +++ b/hummingbot/connector/exchange/huobi/huobi_order_book_tracker.py @@ -78,10 +78,8 @@ async def _order_book_diff_router(self): # Log some statistics. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Diff messages processed: %d, rejected: %d, queued: %d", - messages_accepted, - messages_rejected, - messages_queued) + self.logger().debug(f"Diff messages processed: {messages_accepted}, " + f"rejected: {messages_rejected}, queued: {messages_queued}") messages_accepted = 0 messages_rejected = 0 messages_queued = 0 @@ -114,13 +112,12 @@ async def _track_single_book(self, trading_pair: str): # Output some statistics periodically. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Processed %d order book diffs for %s.", - diff_messages_accepted, trading_pair) + self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") diff_messages_accepted = 0 last_message_timestamp = now elif message.type is OrderBookMessageType.SNAPSHOT: order_book.apply_snapshot(message.bids, message.asks, message.update_id) - self.logger().debug("Processed order book snapshot for %s.", trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise except Exception: diff --git a/hummingbot/connector/exchange/kraken/kraken_order_book_tracker.py b/hummingbot/connector/exchange/kraken/kraken_order_book_tracker.py index 8ac727f7d5..cce7530271 100644 --- a/hummingbot/connector/exchange/kraken/kraken_order_book_tracker.py +++ b/hummingbot/connector/exchange/kraken/kraken_order_book_tracker.py @@ -78,9 +78,7 @@ async def _order_book_diff_router(self): # Log some statistics. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Diff messages processed: %d, rejected: %d", - messages_accepted, - messages_rejected) + self.logger().debug(f"Diff messages processed: {messages_accepted}, rejected: {messages_rejected}") messages_accepted = 0 messages_rejected = 0 diff --git a/hummingbot/connector/exchange/kucoin/kucoin_order_book_tracker.py b/hummingbot/connector/exchange/kucoin/kucoin_order_book_tracker.py index 8b42c7db4f..10764f2b08 100644 --- a/hummingbot/connector/exchange/kucoin/kucoin_order_book_tracker.py +++ b/hummingbot/connector/exchange/kucoin/kucoin_order_book_tracker.py @@ -75,10 +75,8 @@ async def _order_book_diff_router(self): # Log some statistics. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Diff messages processed: %d, rejected: %d, queued: %d", - messages_accepted, - messages_rejected, - messages_queued) + self.logger().debug(f"Diff messages processed: {messages_accepted}, " + f"rejected: {messages_rejected}, queued: {messages_queued}") messages_accepted = 0 messages_rejected = 0 messages_queued = 0 @@ -127,8 +125,7 @@ async def _track_single_book(self, trading_pair: str): # Output some statistics periodically. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Processed %d order book diffs for %s.", - diff_messages_accepted, trading_pair) + self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") diff_messages_accepted = 0 last_message_timestamp = now elif message.type is OrderBookMessageType.SNAPSHOT: @@ -142,7 +139,7 @@ async def _track_single_book(self, trading_pair: str): d_bids, d_asks = active_order_tracker.convert_diff_message_to_order_book_row(diff_message) order_book.apply_diffs(d_bids, d_asks, diff_message.update_id) - self.logger().debug("Processed order book snapshot for %s.", trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise except Exception: diff --git a/hummingbot/connector/exchange/liquid/liquid_order_book_tracker.py b/hummingbot/connector/exchange/liquid/liquid_order_book_tracker.py index 75b2d50f68..d9c11e0278 100644 --- a/hummingbot/connector/exchange/liquid/liquid_order_book_tracker.py +++ b/hummingbot/connector/exchange/liquid/liquid_order_book_tracker.py @@ -80,10 +80,8 @@ async def _order_book_diff_router(self): # Log some statistics. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Diff messages processed: %d, rejected %d, queued: %d", - messages_accepted, - messages_rejected, - messages_queued) + self.logger().debug(f"Diff messages processed: {messages_accepted}, " + f"rejected: {messages_rejected}, queued: {messages_queued}") messages_accepted = 0 messages_rejected = 0 messages_queued = 0 @@ -146,15 +144,14 @@ async def _track_single_book(self, trading_pair: str): # Output some statistics periodically. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Processed %d order book diffs for %s.", - diff_messages_accepted, trading_pair) + self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") diff_messages_accepted = 0 last_message_timestamp = now elif message.type is OrderBookMessageType.SNAPSHOT: past_diffs: List[OrderBookMessage] = list(past_diffs_window) past_diffs_window.append(message) order_book.restore_from_snapshot_and_diffs(message, past_diffs) - self.logger().debug("Processed order book snapshot for %s.", trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise except Exception: diff --git a/hummingbot/connector/exchange/loopring/loopring_order_book_tracker.py b/hummingbot/connector/exchange/loopring/loopring_order_book_tracker.py index 1f43d627d4..0a9decf1da 100644 --- a/hummingbot/connector/exchange/loopring/loopring_order_book_tracker.py +++ b/hummingbot/connector/exchange/loopring/loopring_order_book_tracker.py @@ -90,7 +90,7 @@ async def _track_single_book(self, trading_pair: str): elif message.type is OrderBookMessageType.SNAPSHOT: s_bids, s_asks = active_order_tracker.convert_snapshot_message_to_order_book_row(message) order_book.apply_snapshot(s_bids, s_asks, message.timestamp) - self.logger().debug("Processed order book snapshot for %s.", trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise diff --git a/hummingbot/connector/exchange/okex/okex_order_book_tracker.py b/hummingbot/connector/exchange/okex/okex_order_book_tracker.py index e29a82c743..a09a5731fe 100644 --- a/hummingbot/connector/exchange/okex/okex_order_book_tracker.py +++ b/hummingbot/connector/exchange/okex/okex_order_book_tracker.py @@ -69,10 +69,8 @@ async def _order_book_diff_router(self): # Log some statistics. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Diff messages processed: %d, rejected: %d, queued: %d", - messages_accepted, - messages_rejected, - messages_queued) + self.logger().debug(f"Diff messages processed: {messages_accepted}, " + f"rejected: {messages_rejected}, queued: {messages_queued}") messages_accepted = 0 messages_rejected = 0 messages_queued = 0 @@ -107,13 +105,12 @@ async def _track_single_book(self, trading_pair: str): # Output some statistics periodically. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Processed %d order book diffs for %s.", - diff_messages_accepted, trading_pair) + self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") diff_messages_accepted = 0 last_message_timestamp = now elif message.type is OrderBookMessageType.SNAPSHOT: order_book.apply_snapshot(message.bids, message.asks, message.update_id) - self.logger().debug("Processed order book snapshot for %s.", trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise except Exception: diff --git a/hummingbot/connector/exchange/probit/probit_order_book_tracker.py b/hummingbot/connector/exchange/probit/probit_order_book_tracker.py index e7ac692ba1..cb5f4f9ab4 100644 --- a/hummingbot/connector/exchange/probit/probit_order_book_tracker.py +++ b/hummingbot/connector/exchange/probit/probit_order_book_tracker.py @@ -85,8 +85,7 @@ async def _track_single_book(self, trading_pair: str): # Output some statistics periodically. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Processed %d order book diffs for %s.", - diff_messages_accepted, trading_pair) + self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") diff_messages_accepted = 0 last_message_timestamp = now elif message.type is OrderBookMessageType.SNAPSHOT: @@ -100,7 +99,7 @@ async def _track_single_book(self, trading_pair: str): d_bids, d_asks = probit_utils.convert_diff_message_to_order_book_row(diff_message) order_book.apply_diffs(d_bids, d_asks, diff_message.update_id) - self.logger().debug("Processed order book snapshot for %s.", trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise except Exception: diff --git a/hummingbot/connector/exchange/radar_relay/radar_relay_order_book_tracker.py b/hummingbot/connector/exchange/radar_relay/radar_relay_order_book_tracker.py index 65c6da593f..34c35917e2 100644 --- a/hummingbot/connector/exchange/radar_relay/radar_relay_order_book_tracker.py +++ b/hummingbot/connector/exchange/radar_relay/radar_relay_order_book_tracker.py @@ -128,10 +128,8 @@ async def _order_book_diff_router(self): # Log some statistics. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Diff messages processed: %d, rejected: %d, queued: %d", - messages_accepted, - messages_rejected, - messages_queued) + self.logger().debug(f"Diff messages processed: {messages_accepted}, " + f"rejected: {messages_rejected}, queued: {messages_queued}") messages_accepted = 0 messages_rejected = 0 messages_queued = 0 @@ -179,8 +177,7 @@ async def _track_single_book(self, trading_pair: str): # Output some statistics periodically. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Processed %d order book diffs for %s.", - diff_messages_accepted, trading_pair) + self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") diff_messages_accepted = 0 last_message_timestamp = now elif message.type is OrderBookMessageType.SNAPSHOT: @@ -194,7 +191,7 @@ async def _track_single_book(self, trading_pair: str): d_bids, d_asks = active_order_tracker.convert_diff_message_to_order_book_row(diff_message) order_book.apply_diffs(d_bids, d_asks, diff_message.update_id) - self.logger().debug("Processed order book snapshot for %s.", trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise except Exception: diff --git a/hummingbot/core/data_type/order_book_tracker.py b/hummingbot/core/data_type/order_book_tracker.py index 36ca95179c..b5a92b2d24 100644 --- a/hummingbot/core/data_type/order_book_tracker.py +++ b/hummingbot/core/data_type/order_book_tracker.py @@ -264,14 +264,13 @@ async def _track_single_book(self, trading_pair: str): # Output some statistics periodically. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Processed %d order book diffs for %s.", - diff_messages_accepted, trading_pair) + self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") diff_messages_accepted = 0 last_message_timestamp = now elif message.type is OrderBookMessageType.SNAPSHOT: past_diffs: List[OrderBookMessage] = list(past_diffs_window) order_book.restore_from_snapshot_and_diffs(message, past_diffs) - self.logger().debug("Processed order book snapshot for %s.", trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise except Exception: @@ -307,9 +306,7 @@ async def _emit_trade_event_loop(self): # Log some statistics. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Trade messages processed: %d, rejected: %d", - messages_accepted, - messages_rejected) + self.logger().debug(f"Trade messages processed: {messages_accepted}, rejected: {messages_rejected}") messages_accepted = 0 messages_rejected = 0 diff --git a/hummingbot/core/rate_oracle/rate_oracle.py b/hummingbot/core/rate_oracle/rate_oracle.py index 14b7d31905..5fd847350c 100644 --- a/hummingbot/core/rate_oracle/rate_oracle.py +++ b/hummingbot/core/rate_oracle/rate_oracle.py @@ -19,20 +19,31 @@ class RateOracleSource(Enum): + """ + Supported sources for RateOracle + """ binance = 0 coingecko = 1 class RateOracle(NetworkBase): + """ + RateOracle provides conversion rates for any given pair token symbols in both async and sync fashions. + It achieves this by query URL on a given source for prices and store them, either in cache or as an object member. + The find_rate is then used on these prices to find a rate on a given pair. + """ + # Set these below class members before query for rates source: RateOracleSource = RateOracleSource.binance global_token: str = "USDT" global_token_symbol: str = "$" + _logger: Optional[HummingbotLogger] = None _shared_instance: "RateOracle" = None _shared_client: Optional[aiohttp.ClientSession] = None _cgecko_supported_vs_tokens: List[str] = [] binance_price_url = "https://api.binance.com/api/v3/ticker/bookTicker" + binance_us_price_url = "https://api.binance.us/api/v3/ticker/bookTicker" coingecko_usd_price_url = "https://api.coingecko.com/api/v3/coins/markets?vs_currency={}&order=market_cap_desc" \ "&per_page=250&page={}&sparkline=false" coingecko_supported_vs_tokens_url = "https://api.coingecko.com/api/v3/simple/supported_vs_currencies" @@ -64,6 +75,9 @@ async def _http_client(cls) -> aiohttp.ClientSession: return cls._shared_client async def get_ready(self): + """ + The network is ready when it first successfully get prices for a given source. + """ try: if not self._ready_event.is_set(): await self._ready_event.wait() @@ -79,27 +93,50 @@ def name(self) -> str: @property def prices(self) -> Dict[str, Decimal]: + """ + Actual prices retrieved from URL + """ return self._prices.copy() - def update_interval(self) -> float: - return 1.0 - def rate(self, pair: str) -> Decimal: + """ + Finds a conversion rate for a given symbol, this can be direct or indirect prices as long as it can find a route + to achieve this. + :param pair: A trading pair, e.g. BTC-USDT + :return A conversion rate + """ return find_rate(self._prices, pair) @classmethod async def rate_async(cls, pair: str) -> Decimal: + """ + Finds a conversion rate in an async operation, it is a class method which can be used directly without having to + start the RateOracle network. + :param pair: A trading pair, e.g. BTC-USDT + :return A conversion rate + """ prices = await cls.get_prices() return find_rate(prices, pair) @classmethod async def global_rate(cls, token: str) -> Decimal: + """ + Finds a conversion rate of a given token to a global token + :param token: A token symbol, e.g. BTC + :return A conversion rate + """ prices = await cls.get_prices() pair = token + "-" + cls.global_token return find_rate(prices, pair) @classmethod async def global_value(cls, token: str, amount: Decimal) -> Decimal: + """ + Finds a value of a given token amount in a global token unit + :param token: A token symbol, e.g. BTC + :param amount: An amount of token to be converted to value + :return A value of the token in global token unit + """ rate = await cls.global_rate(token) rate = Decimal("0") if rate is None else rate return amount * rate @@ -115,10 +152,14 @@ async def fetch_price_loop(self): except Exception: self.logger().network(f"Error fetching new prices from {self.source.name}.", exc_info=True, app_warning_msg=f"Couldn't fetch newest prices from {self.source.name}.") - await asyncio.sleep(self.update_interval()) + await asyncio.sleep(1) @classmethod async def get_prices(cls) -> Dict[str, Decimal]: + """ + Fetches prices of a specified source + :return A dictionary of trading pairs and prices + """ if cls.source == RateOracleSource.binance: return await cls.get_binance_prices() elif cls.source == RateOracleSource.coingecko: @@ -129,24 +170,57 @@ async def get_prices(cls) -> Dict[str, Decimal]: @classmethod @async_ttl_cache(ttl=1, maxsize=1) async def get_binance_prices(cls) -> Dict[str, Decimal]: + """ + Fetches Binance prices from binance.com and binance.us where only USD pairs from binance.us prices are added + to the prices dictionary. + :return A dictionary of trading pairs and prices + """ + results = {} + tasks = [cls.get_binance_prices_by_domain(cls.binance_price_url), + cls.get_binance_prices_by_domain(cls.binance_us_price_url, "USD")] + task_results = await safe_gather(*tasks, return_exceptions=True) + for task_result in task_results: + if isinstance(task_result, Exception): + cls.logger().error("Unexpected error while retrieving rates from Coingecko. " + "Check the log file for more info.") + break + else: + results.update(task_result) + return results + + @classmethod + async def get_binance_prices_by_domain(cls, url: str, quote_symbol: str = None) -> Dict[str, Decimal]: + """ + Fetches binance prices + :param url: A URL end point + :param quote_symbol: A quote symbol, if specified only pairs with the quote symbol are included for prices + :return A dictionary of trading pairs and prices + """ results = {} client = await cls._http_client() - try: - async with client.request("GET", cls.binance_price_url) as resp: - records = await resp.json() - for record in records: - trading_pair = binance_convert_from_exchange_pair(record["symbol"]) - if trading_pair and record["bidPrice"] is not None and record["askPrice"] is not None: - results[trading_pair] = (Decimal(record["bidPrice"]) + Decimal(record["askPrice"])) / Decimal("2") - except asyncio.CancelledError: - raise - except Exception: - cls.logger().error("Unexpected error while retrieving rates from Binance.") + async with client.request("GET", url) as resp: + records = await resp.json() + for record in records: + trading_pair = binance_convert_from_exchange_pair(record["symbol"]) + if quote_symbol is not None: + base, quote = trading_pair.split("-") + if quote != quote_symbol: + continue + if trading_pair and record["bidPrice"] is not None and record["askPrice"] is not None: + results[trading_pair] = (Decimal(record["bidPrice"]) + Decimal(record["askPrice"])) / Decimal( + "2") return results @classmethod @async_ttl_cache(ttl=30, maxsize=1) async def get_coingecko_prices(cls, vs_currency: str) -> Dict[str, Decimal]: + """ + Fetches CoinGecko prices for the top 1000 token (order by market cap), each API query returns 250 results, + hence it queries 4 times concurrently. + :param vs_currency: A currency (crypto or fiat) to get prices of tokens in, see + https://api.coingecko.com/api/v3/simple/supported_vs_currencies for the current supported list + :return A dictionary of trading pairs and prices + """ results = {} if not cls._cgecko_supported_vs_tokens: client = await cls._http_client() @@ -168,6 +242,13 @@ async def get_coingecko_prices(cls, vs_currency: str) -> Dict[str, Decimal]: @classmethod async def get_coingecko_prices_by_page(cls, vs_currency: str, page_no: int) -> Dict[str, Decimal]: + """ + Fetches CoinGecko prices by page number. + :param vs_currency: A currency (crypto or fiat) to get prices of tokens in, see + https://api.coingecko.com/api/v3/simple/supported_vs_currencies for the current supported list + :param page_no: The page number + :return A dictionary of trading pairs and prices (250 results max) + """ results = {} client = await cls._http_client() async with client.request("GET", cls.coingecko_usd_price_url.format(vs_currency, page_no)) as resp: diff --git a/hummingbot/core/rate_oracle/utils.py b/hummingbot/core/rate_oracle/utils.py index 0192705764..261c6dae79 100644 --- a/hummingbot/core/rate_oracle/utils.py +++ b/hummingbot/core/rate_oracle/utils.py @@ -16,6 +16,8 @@ def find_rate(prices: Dict[str, Decimal], pair: str) -> Decimal: if pair in prices: return prices[pair] base, quote = pair.split("-") + if base == quote: + return Decimal("1") reverse_pair = f"{quote}-{base}" if reverse_pair in prices: return Decimal("1") / prices[reverse_pair] diff --git a/hummingbot/core/utils/estimate_fee.py b/hummingbot/core/utils/estimate_fee.py index 78823e425e..5a52568861 100644 --- a/hummingbot/core/utils/estimate_fee.py +++ b/hummingbot/core/utils/estimate_fee.py @@ -2,16 +2,11 @@ from hummingbot.core.event.events import TradeFee, TradeFeeType from hummingbot.client.config.fee_overrides_config_map import fee_overrides_config_map from hummingbot.client.settings import CONNECTOR_SETTINGS -from hummingbot.core.utils.eth_gas_station_lookup import get_gas_price, get_gas_limit def estimate_fee(exchange: str, is_maker: bool) -> TradeFee: if exchange not in CONNECTOR_SETTINGS: raise Exception(f"Invalid connector. {exchange} does not exist in CONNECTOR_SETTINGS") - use_gas = CONNECTOR_SETTINGS[exchange].use_eth_gas_lookup - if use_gas: - gas_amount = get_gas_price(in_gwei=False) * get_gas_limit(exchange) - return TradeFee(percent=0, flat_fees=[("ETH", gas_amount)]) fee_type = CONNECTOR_SETTINGS[exchange].fee_type fee_token = CONNECTOR_SETTINGS[exchange].fee_token default_fees = CONNECTOR_SETTINGS[exchange].default_fees diff --git a/hummingbot/core/utils/eth_gas_station_lookup.py b/hummingbot/core/utils/eth_gas_station_lookup.py deleted file mode 100644 index ee56502631..0000000000 --- a/hummingbot/core/utils/eth_gas_station_lookup.py +++ /dev/null @@ -1,178 +0,0 @@ -import asyncio -import requests -import logging -from typing import ( - Optional, - Dict, - Any -) -import aiohttp -from enum import Enum -from decimal import Decimal -from hummingbot.core.network_base import ( - NetworkBase, - NetworkStatus -) -from hummingbot.core.utils.async_utils import safe_ensure_future -from hummingbot.logger import HummingbotLogger -from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.client.settings import CONNECTOR_SETTINGS -from hummingbot.client.settings import GATEAWAY_CA_CERT_PATH, GATEAWAY_CLIENT_CERT_PATH, GATEAWAY_CLIENT_KEY_PATH - -ETH_GASSTATION_API_URL = "https://data-api.defipulse.com/api/v1/egs/api/ethgasAPI.json?api-key={}" - - -def get_gas_price(in_gwei: bool = True) -> Decimal: - if not global_config_map["ethgasstation_gas_enabled"].value: - gas_price = global_config_map["manual_gas_price"].value - else: - gas_price = EthGasStationLookup.get_instance().gas_price - return gas_price if in_gwei else gas_price / Decimal("1e9") - - -def get_gas_limit(connector_name: str) -> int: - gas_limit = request_gas_limit(connector_name) - return gas_limit - - -def request_gas_limit(connector_name: str) -> int: - host = global_config_map["gateway_api_host"].value - port = global_config_map["gateway_api_port"].value - balancer_max_swaps = global_config_map["balancer_max_swaps"].value - - base_url = ':'.join(['https://' + host, port]) - url = f"{base_url}/{connector_name}/gas-limit" - - ca_certs = GATEAWAY_CA_CERT_PATH - client_certs = (GATEAWAY_CLIENT_CERT_PATH, GATEAWAY_CLIENT_KEY_PATH) - params = {"maxSwaps": balancer_max_swaps} if connector_name == "balancer" else {} - response = requests.post(url, data=params, verify=ca_certs, cert=client_certs) - parsed_response = response.json() - if response.status_code != 200: - err_msg = "" - if "error" in parsed_response: - err_msg = f" Message: {parsed_response['error']}" - raise IOError(f"Error fetching data from {url}. HTTP status is {response.status}.{err_msg}") - if "error" in parsed_response: - raise Exception(f"Error: {parsed_response['error']}") - return parsed_response['gasLimit'] - - -class GasLevel(Enum): - fast = "fast" - fastest = "fastest" - safeLow = "safeLow" - average = "average" - - -class EthGasStationLookup(NetworkBase): - _egsl_logger: Optional[HummingbotLogger] = None - _shared_instance: "EthGasStationLookup" = None - - @classmethod - def get_instance(cls) -> "EthGasStationLookup": - if cls._shared_instance is None: - cls._shared_instance = EthGasStationLookup() - return cls._shared_instance - - @classmethod - def logger(cls) -> HummingbotLogger: - if cls._egsl_logger is None: - cls._egsl_logger = logging.getLogger(__name__) - return cls._egsl_logger - - def __init__(self): - super().__init__() - self._gas_prices: Dict[str, Decimal] = {} - self._gas_limits: Dict[str, Decimal] = {} - self._balancer_max_swaps: int = global_config_map["balancer_max_swaps"].value - self._async_task = None - - @property - def api_key(self): - return global_config_map["ethgasstation_api_key"].value - - @property - def gas_level(self) -> GasLevel: - return GasLevel[global_config_map["ethgasstation_gas_level"].value] - - @property - def refresh_time(self): - return global_config_map["ethgasstation_refresh_time"].value - - @property - def gas_price(self): - return self._gas_prices[self.gas_level] - - @property - def gas_limits(self): - return self._gas_limits - - @gas_limits.setter - def gas_limits(self, gas_limits: Dict[str, int]): - for key, value in gas_limits.items(): - self._gas_limits[key] = value - - @property - def balancer_max_swaps(self): - return self._balancer_max_swaps - - @balancer_max_swaps.setter - def balancer_max_swaps(self, max_swaps: int): - self._balancer_max_swaps = max_swaps - - async def gas_price_update_loop(self): - while True: - try: - url = ETH_GASSTATION_API_URL.format(self.api_key) - async with aiohttp.ClientSession() as client: - response = await client.get(url=url) - if response.status != 200: - raise IOError(f"Error fetching current gas prices. " - f"HTTP status is {response.status}.") - resp_data: Dict[str, Any] = await response.json() - for key, value in resp_data.items(): - if key in GasLevel.__members__: - self._gas_prices[GasLevel[key]] = Decimal(str(value)) / Decimal("10") - prices_str = ', '.join([k.name + ': ' + str(v) for k, v in self._gas_prices.items()]) - self.logger().info(f"Gas levels: [{prices_str}]") - for name, con_setting in CONNECTOR_SETTINGS.items(): - if con_setting.use_eth_gas_lookup: - self._gas_limits[name] = get_gas_limit(name) - self.logger().info(f"{name} Gas estimate:" - f" limit = {self._gas_limits[name]:.0f}," - f" price = {self.gas_level.name}," - f" estimated cost = {get_gas_price(False) * self._gas_limits[name]:.5f} ETH") - await asyncio.sleep(self.refresh_time) - except asyncio.CancelledError: - raise - except Exception: - self.logger().network("Unexpected error running logging task.", exc_info=True) - await asyncio.sleep(self.refresh_time) - - async def start_network(self): - self._async_task = safe_ensure_future(self.gas_price_update_loop()) - - async def stop_network(self): - if self._async_task is not None: - self._async_task.cancel() - self._async_task = None - - async def check_network(self) -> NetworkStatus: - try: - url = ETH_GASSTATION_API_URL.format(self.api_key) - async with aiohttp.ClientSession() as client: - response = await client.get(url=url) - if response.status != 200: - raise Exception(f"Error connecting to {url}. HTTP status is {response.status}.") - except asyncio.CancelledError: - raise - except Exception: - return NetworkStatus.NOT_CONNECTED - return NetworkStatus.CONNECTED - - def start(self): - NetworkBase.start(self) - - def stop(self): - NetworkBase.stop(self) diff --git a/hummingbot/core/utils/ethereum.py b/hummingbot/core/utils/ethereum.py index ff667d3a0a..e30c2b3478 100644 --- a/hummingbot/core/utils/ethereum.py +++ b/hummingbot/core/utils/ethereum.py @@ -3,7 +3,11 @@ from hexbytes import HexBytes from web3 import Web3 from web3.datastructures import AttributeDict -from typing import Dict +from typing import Dict, List +import aiohttp +from hummingbot.client.config.global_config_map import global_config_map +import itertools as it +from hummingbot.core.utils import async_ttl_cache def check_web3(ethereum_rpc_url: str) -> bool: @@ -32,3 +36,55 @@ def block_values_to_hex(block: AttributeDict) -> AttributeDict: except binascii.Error: formatted_block[key] = value return AttributeDict(formatted_block) + + +def check_transaction_exceptions(trade_data: dict) -> dict: + + exception_list = [] + + gas_limit = trade_data["gas_limit"] + # gas_price = trade_data["gas_price"] + gas_cost = trade_data["gas_cost"] + amount = trade_data["amount"] + side = trade_data["side"] + base = trade_data["base"] + quote = trade_data["quote"] + balances = trade_data["balances"] + allowances = trade_data["allowances"] + swaps_message = f"Total swaps: {trade_data['swaps']}" if "swaps" in trade_data.keys() else '' + + eth_balance = balances["ETH"] + + # check for sufficient gas + if eth_balance < gas_cost: + exception_list.append(f"Insufficient ETH balance to cover gas:" + f" Balance: {eth_balance}. Est. gas cost: {gas_cost}. {swaps_message}") + + trade_token = base if side == "side" else quote + trade_allowance = allowances[trade_token] + + # check for gas limit set to low + gas_limit_threshold = 21000 + if gas_limit < gas_limit_threshold: + exception_list.append(f"Gas limit {gas_limit} below recommended {gas_limit_threshold} threshold.") + + # check for insufficient token allowance + if allowances[trade_token] < amount: + exception_list.append(f"Insufficient {trade_token} allowance {trade_allowance}. Amount to trade: {amount}") + + return exception_list + + +@async_ttl_cache(ttl=30) +async def fetch_trading_pairs() -> List[str]: + token_list_url = global_config_map.get("ethereum_token_list_url").value + tokens = set() + async with aiohttp.ClientSession() as client: + resp = await client.get(token_list_url) + resp_json = await resp.json() + for token in resp_json["tokens"]: + tokens.add(token["symbol"]) + trading_pairs = [] + for base, quote in it.permutations(tokens, 2): + trading_pairs.append(f"{base}-{quote}") + return trading_pairs diff --git a/hummingbot/core/utils/trading_pair_fetcher.py b/hummingbot/core/utils/trading_pair_fetcher.py index ea07080568..1da79fa514 100644 --- a/hummingbot/core/utils/trading_pair_fetcher.py +++ b/hummingbot/core/utils/trading_pair_fetcher.py @@ -4,11 +4,11 @@ Any, Optional, ) +from hummingbot.core.utils.async_utils import safe_gather from hummingbot.logger import HummingbotLogger from hummingbot.client.settings import CONNECTOR_SETTINGS, ConnectorType import logging import asyncio -import requests from .async_utils import safe_ensure_future @@ -35,6 +35,8 @@ def __init__(self): safe_ensure_future(self.fetch_all()) async def fetch_all(self): + tasks = [] + fetched_connectors = [] for conn_setting in CONNECTOR_SETTINGS.values(): module_name = f"{conn_setting.base_name()}_connector" if conn_setting.type is ConnectorType.Connector \ else f"{conn_setting.base_name()}_api_order_book_data_source" @@ -46,15 +48,13 @@ async def fetch_all(self): module = getattr(importlib.import_module(module_path), class_name) args = {} args = conn_setting.add_domain_parameter(args) - safe_ensure_future(self.call_fetch_pairs(module.fetch_trading_pairs(**args), conn_setting.name)) + tasks.append(asyncio.wait_for(asyncio.shield(module.fetch_trading_pairs(**args)), timeout=3)) + fetched_connectors.append(conn_setting.name) - self.ready = True - - async def call_fetch_pairs(self, fetch_fn, exchange_name): + results = await safe_gather(*tasks, return_exceptions=True) + self.trading_pairs = dict(zip(fetched_connectors, results)) # In case trading pair fetching returned timeout, using empty list - try: - self.trading_pairs[exchange_name] = await fetch_fn - except (asyncio.TimeoutError, asyncio.CancelledError, requests.exceptions.RequestException): - self.logger().error(f"Connector {exchange_name} failed to retrieve its trading pairs. " - f"Trading pairs autocompletion won't work.") - self.trading_pairs[exchange_name] = [] + for connector, result in self.trading_pairs.items(): + if isinstance(result, asyncio.TimeoutError): + self.trading_pairs[connector] = [] + self.ready = True diff --git a/hummingbot/strategy/__utils__/trailing_indicators/average_volatility.py b/hummingbot/strategy/__utils__/trailing_indicators/average_volatility.py new file mode 100644 index 0000000000..cf57615e80 --- /dev/null +++ b/hummingbot/strategy/__utils__/trailing_indicators/average_volatility.py @@ -0,0 +1,14 @@ +from .base_trailing_indicator import BaseTrailingIndicator +import numpy as np + + +class AverageVolatilityIndicator(BaseTrailingIndicator): + def __init__(self, sampling_length: int = 30, processing_length: int = 15): + super().__init__(sampling_length, processing_length) + + def _indicator_calculation(self) -> float: + return np.var(self._sampling_buffer.get_as_numpy_array()) + + def _processing_calculation(self) -> float: + processing_array = self._processing_buffer.get_as_numpy_array() + return np.sqrt(np.mean(processing_array)) diff --git a/hummingbot/strategy/__utils__/trailing_indicators/base_trailing_indicator.py b/hummingbot/strategy/__utils__/trailing_indicators/base_trailing_indicator.py new file mode 100644 index 0000000000..8df46b58b0 --- /dev/null +++ b/hummingbot/strategy/__utils__/trailing_indicators/base_trailing_indicator.py @@ -0,0 +1,49 @@ +from abc import ABC, abstractmethod +import numpy as np +import logging +from ..ring_buffer import RingBuffer + +pmm_logger = None + + +class BaseTrailingIndicator(ABC): + @classmethod + def logger(cls): + global pmm_logger + if pmm_logger is None: + pmm_logger = logging.getLogger(__name__) + return pmm_logger + + def __init__(self, sampling_length: int = 30, processing_length: int = 15): + self._sampling_length = sampling_length + self._sampling_buffer = RingBuffer(sampling_length) + self._processing_length = processing_length + self._processing_buffer = RingBuffer(processing_length) + + def add_sample(self, value: float): + self._sampling_buffer.add_value(value) + indicator_value = self._indicator_calculation() + self._processing_buffer.add_value(indicator_value) + + @abstractmethod + def _indicator_calculation(self) -> float: + raise NotImplementedError + + def _processing_calculation(self) -> float: + """ + Processing of the processing buffer to return final value. + Default behavior is buffer average + """ + return np.mean(self._processing_buffer.get_as_numpy_array()) + + @property + def current_value(self) -> float: + return self._processing_calculation() + + @property + def is_sampling_buffer_full(self) -> bool: + return self._sampling_buffer.is_full + + @property + def is_processing_buffer_full(self) -> bool: + return self._processing_buffer.is_full diff --git a/hummingbot/strategy/__utils__/trailing_indicators/exponential_moving_average.py b/hummingbot/strategy/__utils__/trailing_indicators/exponential_moving_average.py new file mode 100644 index 0000000000..ed380fc99a --- /dev/null +++ b/hummingbot/strategy/__utils__/trailing_indicators/exponential_moving_average.py @@ -0,0 +1,17 @@ +from base_trailing_indicator import BaseTrailingIndicator +import pandas as pd + + +class ExponentialMovingAverageIndicator(BaseTrailingIndicator): + def __init__(self, sampling_length: int = 30, processing_length: int = 1): + if processing_length != 1: + raise Exception("Exponential moving average processing_length should be 1") + super().__init__(sampling_length, processing_length) + + def _indicator_calculation(self) -> float: + ema = pd.Series(self._sampling_buffer.get_as_numpy_array())\ + .ewm(span=self._sampling_length, adjust=True).mean() + return ema[-1] + + def _processing_calculation(self) -> float: + return self._processing_buffer.get_last_value() diff --git a/hummingbot/strategy/amm_arb/amm_arb.py b/hummingbot/strategy/amm_arb/amm_arb.py index b96af808fc..ac3e620419 100644 --- a/hummingbot/strategy/amm_arb/amm_arb.py +++ b/hummingbot/strategy/amm_arb/amm_arb.py @@ -12,6 +12,7 @@ from hummingbot.strategy.strategy_py_base import StrategyPyBase from hummingbot.connector.connector_base import ConnectorBase from hummingbot.client.settings import ETH_WALLET_CONNECTORS +from hummingbot.client.performance import smart_round from hummingbot.connector.connector.uniswap.uniswap_connector import UniswapConnector from .utils import create_arb_proposals, ArbProposal @@ -253,13 +254,18 @@ async def format_status(self) -> str: market, trading_pair, base_asset, quote_asset = market_info buy_price = await market.get_quote_price(trading_pair, True, self._order_amount) sell_price = await market.get_quote_price(trading_pair, False, self._order_amount) - mid_price = (buy_price + sell_price) / 2 + + # check for unavailable price data + buy_price = smart_round(Decimal(str(buy_price)), 8) if buy_price is not None else '-' + sell_price = smart_round(Decimal(str(sell_price)), 8) if sell_price is not None else '-' + mid_price = smart_round(((buy_price + sell_price) / 2), 8) if '-' not in [buy_price, sell_price] else '-' + data.append([ market.display_name, trading_pair, - float(sell_price), - float(buy_price), - float(mid_price) + sell_price, + buy_price, + mid_price ]) markets_df = pd.DataFrame(data=data, columns=columns) lines = [] @@ -335,7 +341,7 @@ async def quote_in_eth_rate_fetch_loop(self): self._market_2_quote_eth_rate = await self.request_rate_in_eth(self._market_info_2.quote_asset) self.logger().warning(f"Estimate conversion rate - " f"{self._market_info_2.quote_asset}:ETH = {self._market_2_quote_eth_rate} ") - await asyncio.sleep(60 * 5) + await asyncio.sleep(60 * 1) except asyncio.CancelledError: raise except Exception as e: @@ -348,4 +354,5 @@ async def quote_in_eth_rate_fetch_loop(self): async def request_rate_in_eth(self, quote: str) -> int: if self._uniswap is None: self._uniswap = UniswapConnector([f"{quote}-WETH"], "", None) + await self._uniswap.initiate_pool() # initiate to cache swap pool return await self._uniswap.get_quote_price(f"{quote}-WETH", True, 1) diff --git a/hummingbot/strategy/arbitrage/arbitrage.pxd b/hummingbot/strategy/arbitrage/arbitrage.pxd index 316fd6ede6..9d999eea9d 100755 --- a/hummingbot/strategy/arbitrage/arbitrage.pxd +++ b/hummingbot/strategy/arbitrage/arbitrage.pxd @@ -23,10 +23,12 @@ cdef class ArbitrageStrategy(StrategyBase): object _exchange_rate_conversion int _failed_order_tolerance bint _cool_off_logged + bint _use_oracle_conversion_rate object _secondary_to_primary_base_conversion_rate object _secondary_to_primary_quote_conversion_rate bint _hb_app_notification tuple _current_profitability + double _last_conv_rates_logged cdef tuple c_calculate_arbitrage_top_order_profitability(self, object market_pair) cdef c_process_market_pair(self, object market_pair) diff --git a/hummingbot/strategy/arbitrage/arbitrage.pyx b/hummingbot/strategy/arbitrage/arbitrage.pyx index d08a6afc83..b406d9a742 100755 --- a/hummingbot/strategy/arbitrage/arbitrage.pyx +++ b/hummingbot/strategy/arbitrage/arbitrage.pyx @@ -20,6 +20,8 @@ from hummingbot.core.network_iterator import NetworkStatus from hummingbot.strategy.strategy_base import StrategyBase from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple from hummingbot.strategy.arbitrage.arbitrage_market_pair import ArbitrageMarketPair +from hummingbot.core.rate_oracle.rate_oracle import RateOracle +from hummingbot.client.performance import smart_round NaN = float("nan") s_decimal_0 = Decimal(0) @@ -49,6 +51,7 @@ cdef class ArbitrageStrategy(StrategyBase): status_report_interval: float = 60.0, next_trade_delay_interval: float = 15.0, failed_order_tolerance: int = 1, + use_oracle_conversion_rate: bool = False, secondary_to_primary_base_conversion_rate: Decimal = Decimal("1"), secondary_to_primary_quote_conversion_rate: Decimal = Decimal("1"), hb_app_notification: bool = False): @@ -75,9 +78,10 @@ cdef class ArbitrageStrategy(StrategyBase): self._failed_order_tolerance = failed_order_tolerance self._cool_off_logged = False self._current_profitability = () - + self._use_oracle_conversion_rate = use_oracle_conversion_rate self._secondary_to_primary_base_conversion_rate = secondary_to_primary_base_conversion_rate self._secondary_to_primary_quote_conversion_rate = secondary_to_primary_quote_conversion_rate + self._last_conv_rates_logged = 0 self._hb_app_notification = hb_app_notification @@ -106,6 +110,55 @@ cdef class ArbitrageStrategy(StrategyBase): def tracked_market_orders_data_frame(self) -> List[pd.DataFrame]: return self._sb_order_tracker.tracked_market_orders_data_frame + def get_second_to_first_conversion_rate(self) -> Tuple[str, Decimal, str, Decimal]: + """ + Find conversion rates from secondary market to primary market + :return: A tuple of quote pair symbol, quote conversion rate source, quote conversion rate, + base pair symbol, base conversion rate source, base conversion rate + """ + quote_rate = Decimal("1") + quote_pair = f"{self._market_pairs[0].second.quote_asset}-{self._market_pairs[0].first.quote_asset}" + quote_rate_source = "fixed" + if self._use_oracle_conversion_rate: + if self._market_pairs[0].second.quote_asset != self._market_pairs[0].first.quote_asset: + quote_rate_source = RateOracle.source.name + quote_rate = RateOracle.get_instance().rate(quote_pair) + else: + quote_rate = self._secondary_to_primary_quote_conversion_rate + base_rate = Decimal("1") + base_pair = f"{self._market_pairs[0].second.base_asset}-{self._market_pairs[0].first.base_asset}" + base_rate_source = "fixed" + if self._use_oracle_conversion_rate: + if self._market_pairs[0].second.base_asset != self._market_pairs[0].first.base_asset: + base_rate_source = RateOracle.source.name + base_rate = RateOracle.get_instance().rate(base_pair) + else: + base_rate = self._secondary_to_primary_base_conversion_rate + return quote_pair, quote_rate_source, quote_rate, base_pair, base_rate_source, base_rate + + def log_conversion_rates(self): + quote_pair, quote_rate_source, quote_rate, base_pair, base_rate_source, base_rate = \ + self.get_second_to_first_conversion_rate() + if quote_pair.split("-")[0] != quote_pair.split("-")[1]: + self.logger().info(f"{quote_pair} ({quote_rate_source}) conversion rate: {smart_round(quote_rate)}") + if base_pair.split("-")[0] != base_pair.split("-")[1]: + self.logger().info(f"{base_pair} ({base_rate_source}) conversion rate: {smart_round(base_rate)}") + + def oracle_status_df(self): + columns = ["Source", "Pair", "Rate"] + data = [] + quote_pair, quote_rate_source, quote_rate, base_pair, base_rate_source, base_rate = \ + self.get_second_to_first_conversion_rate() + if quote_pair.split("-")[0] != quote_pair.split("-")[1]: + data.extend([ + [quote_rate_source, quote_pair, smart_round(quote_rate)], + ]) + if base_pair.split("-")[0] != base_pair.split("-")[1]: + data.extend([ + [base_rate_source, base_pair, smart_round(base_rate)], + ]) + return pd.DataFrame(data=data, columns=columns) + def format_status(self) -> str: cdef: list lines = [] @@ -117,6 +170,11 @@ cdef class ArbitrageStrategy(StrategyBase): lines.extend(["", " Markets:"] + [" " + line for line in str(markets_df).split("\n")]) + oracle_df = self.oracle_status_df() + if not oracle_df.empty: + lines.extend(["", " Rate conversion:"] + + [" " + line for line in str(oracle_df).split("\n")]) + assets_df = self.wallet_balance_data_frame([market_pair.first, market_pair.second]) lines.extend(["", " Assets:"] + [" " + line for line in str(assets_df).split("\n")]) @@ -194,6 +252,10 @@ cdef class ArbitrageStrategy(StrategyBase): for market_pair in self._market_pairs: self.c_process_market_pair(market_pair) + # log conversion rates every 5 minutes + if self._last_conv_rates_logged + (60. * 5) < self._current_timestamp: + self.log_conversion_rates() + self._last_conv_rates_logged = self._current_timestamp finally: self._last_timestamp = timestamp @@ -390,7 +452,20 @@ cdef class ArbitrageStrategy(StrategyBase): if market_info == self._market_pairs[0].first: return Decimal("1") elif market_info == self._market_pairs[0].second: - return self._secondary_to_primary_quote_conversion_rate / self._secondary_to_primary_base_conversion_rate + _, _, quote_rate, _, _, base_rate = self.get_second_to_first_conversion_rate() + return quote_rate / base_rate + # if not self._use_oracle_conversion_rate: + # return self._secondary_to_primary_quote_conversion_rate / self._secondary_to_primary_base_conversion_rate + # else: + # quote_rate = Decimal("1") + # if self._market_pairs[0].second.quote_asset != self._market_pairs[0].first.quote_asset: + # quote_pair = f"{self._market_pairs[0].second.quote_asset}-{self._market_pairs[0].first.quote_asset}" + # quote_rate = RateOracle.get_instance().rate(quote_pair) + # base_rate = Decimal("1") + # if self._market_pairs[0].second.base_asset != self._market_pairs[0].first.base_asset: + # base_pair = f"{self._market_pairs[0].second.base_asset}-{self._market_pairs[0].first.base_asset}" + # base_rate = RateOracle.get_instance().rate(base_pair) + # return quote_rate / base_rate cdef tuple c_find_best_profitable_amount(self, object buy_market_trading_pair_tuple, object sell_market_trading_pair_tuple): """ diff --git a/hummingbot/strategy/arbitrage/arbitrage_config_map.py b/hummingbot/strategy/arbitrage/arbitrage_config_map.py index 96313098ae..781396f642 100644 --- a/hummingbot/strategy/arbitrage/arbitrage_config_map.py +++ b/hummingbot/strategy/arbitrage/arbitrage_config_map.py @@ -2,12 +2,11 @@ from hummingbot.client.config.config_validators import ( validate_exchange, validate_market_trading_pair, - validate_decimal -) -from hummingbot.client.settings import ( - required_exchanges, - EXAMPLE_PAIRS, + validate_decimal, + validate_bool ) +from hummingbot.client.config.config_helpers import parse_cvar_value +import hummingbot.client.settings as settings from decimal import Decimal from typing import Optional @@ -24,70 +23,109 @@ def validate_secondary_market_trading_pair(value: str) -> Optional[str]: def primary_trading_pair_prompt(): primary_market = arbitrage_config_map.get("primary_market").value - example = EXAMPLE_PAIRS.get(primary_market) + example = settings.EXAMPLE_PAIRS.get(primary_market) return "Enter the token trading pair you would like to trade on %s%s >>> " \ % (primary_market, f" (e.g. {example})" if example else "") def secondary_trading_pair_prompt(): secondary_market = arbitrage_config_map.get("secondary_market").value - example = EXAMPLE_PAIRS.get(secondary_market) + example = settings.EXAMPLE_PAIRS.get(secondary_market) return "Enter the token trading pair you would like to trade on %s%s >>> " \ % (secondary_market, f" (e.g. {example})" if example else "") def secondary_market_on_validated(value: str): - required_exchanges.append(value) + settings.required_exchanges.append(value) + + +def update_oracle_settings(value: str): + c_map = arbitrage_config_map + if not (c_map["use_oracle_conversion_rate"].value is not None and + c_map["primary_market_trading_pair"].value is not None and + c_map["secondary_market_trading_pair"].value is not None): + return + use_oracle = parse_cvar_value(c_map["use_oracle_conversion_rate"], c_map["use_oracle_conversion_rate"].value) + first_base, first_quote = c_map["primary_market_trading_pair"].value.split("-") + second_base, second_quote = c_map["secondary_market_trading_pair"].value.split("-") + if use_oracle and (first_base != second_base or first_quote != second_quote): + settings.required_rate_oracle = True + settings.rate_oracle_pairs = [] + if first_base != second_base: + settings.rate_oracle_pairs.append(f"{second_base}-{first_base}") + if first_quote != second_quote: + settings.rate_oracle_pairs.append(f"{second_quote}-{first_quote}") + else: + settings.required_rate_oracle = False + settings.rate_oracle_pairs = [] arbitrage_config_map = { - "strategy": - ConfigVar(key="strategy", - prompt="", - default="arbitrage"), + "strategy": ConfigVar( + key="strategy", + prompt="", + default="arbitrage" + ), "primary_market": ConfigVar( key="primary_market", prompt="Enter your primary spot connector >>> ", prompt_on_new=True, validator=validate_exchange, - on_validated=lambda value: required_exchanges.append(value)), + on_validated=lambda value: settings.required_exchanges.append(value), + ), "secondary_market": ConfigVar( key="secondary_market", prompt="Enter your secondary spot connector >>> ", prompt_on_new=True, validator=validate_exchange, - on_validated=secondary_market_on_validated), + on_validated=secondary_market_on_validated, + ), "primary_market_trading_pair": ConfigVar( key="primary_market_trading_pair", prompt=primary_trading_pair_prompt, prompt_on_new=True, - validator=validate_primary_market_trading_pair), + validator=validate_primary_market_trading_pair, + on_validated=update_oracle_settings, + ), "secondary_market_trading_pair": ConfigVar( key="secondary_market_trading_pair", prompt=secondary_trading_pair_prompt, prompt_on_new=True, - validator=validate_secondary_market_trading_pair), + validator=validate_secondary_market_trading_pair, + on_validated=update_oracle_settings, + ), "min_profitability": ConfigVar( key="min_profitability", prompt="What is the minimum profitability for you to make a trade? (Enter 1 to indicate 1%) >>> ", prompt_on_new=True, default=Decimal("0.3"), validator=lambda v: validate_decimal(v, Decimal(-100), Decimal("100"), inclusive=True), - type_str="decimal"), + type_str="decimal", + ), + "use_oracle_conversion_rate": ConfigVar( + key="use_oracle_conversion_rate", + type_str="bool", + prompt="Do you want to use rate oracle on unmatched trading pairs? (Yes/No) >>> ", + prompt_on_new=True, + validator=lambda v: validate_bool(v), + on_validated=update_oracle_settings, + ), "secondary_to_primary_base_conversion_rate": ConfigVar( key="secondary_to_primary_base_conversion_rate", prompt="Enter conversion rate for secondary base asset value to primary base asset value, e.g. " - "if primary base asset is USD, secondary is DAI and 1 USD is worth 1.25 DAI, " - "the conversion rate is 0.8 (1 / 1.25) >>> ", + "if primary base asset is USD and the secondary is DAI, 1 DAI is valued at 1.25 USD, " + "the conversion rate is 1.25 >>> ", default=Decimal("1"), - validator=lambda v: validate_decimal(v, Decimal(0), Decimal("100"), inclusive=False), - type_str="decimal"), + validator=lambda v: validate_decimal(v, Decimal(0), inclusive=False), + type_str="decimal", + ), "secondary_to_primary_quote_conversion_rate": ConfigVar( key="secondary_to_primary_quote_conversion_rate", prompt="Enter conversion rate for secondary quote asset value to primary quote asset value, e.g. " - "if primary quote asset is USD, secondary is DAI and 1 USD is worth 1.25 DAI, " - "the conversion rate is 0.8 (1 / 1.25) >>> ", + "if primary quote asset is USD and the secondary is DAI and 1 DAI is valued at 1.25 USD, " + "the conversion rate is 1.25 >>> ", default=Decimal("1"), - validator=lambda v: validate_decimal(v, Decimal(0), Decimal("100"), inclusive=False), - type_str="decimal"), + validator=lambda v: validate_decimal(v, Decimal(0), inclusive=False), + type_str="decimal", + ), } diff --git a/hummingbot/strategy/arbitrage/start.py b/hummingbot/strategy/arbitrage/start.py index e9afc4f3ba..b429e8cb89 100644 --- a/hummingbot/strategy/arbitrage/start.py +++ b/hummingbot/strategy/arbitrage/start.py @@ -15,6 +15,7 @@ def start(self): raw_primary_trading_pair = arbitrage_config_map.get("primary_market_trading_pair").value raw_secondary_trading_pair = arbitrage_config_map.get("secondary_market_trading_pair").value min_profitability = arbitrage_config_map.get("min_profitability").value / Decimal("100") + use_oracle_conversion_rate = arbitrage_config_map.get("use_oracle_conversion_rate").value secondary_to_primary_base_conversion_rate = arbitrage_config_map["secondary_to_primary_base_conversion_rate"].value secondary_to_primary_quote_conversion_rate = arbitrage_config_map["secondary_to_primary_quote_conversion_rate"].value @@ -41,6 +42,7 @@ def start(self): self.strategy = ArbitrageStrategy(market_pairs=[self.market_pair], min_profitability=min_profitability, logging_options=ArbitrageStrategy.OPTION_LOG_ALL, + use_oracle_conversion_rate=use_oracle_conversion_rate, secondary_to_primary_base_conversion_rate=secondary_to_primary_base_conversion_rate, secondary_to_primary_quote_conversion_rate=secondary_to_primary_quote_conversion_rate, hb_app_notification=True) diff --git a/hummingbot/strategy/avellaneda_market_making/__init__.py b/hummingbot/strategy/avellaneda_market_making/__init__.py new file mode 100644 index 0000000000..d29aaf1e02 --- /dev/null +++ b/hummingbot/strategy/avellaneda_market_making/__init__.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python + +from .avellaneda_market_making import AvellanedaMarketMakingStrategy +__all__ = [ + AvellanedaMarketMakingStrategy, +] diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pxd b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pxd new file mode 100644 index 0000000000..79df1f715f --- /dev/null +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pxd @@ -0,0 +1,73 @@ +# distutils: language=c++ + +from libc.stdint cimport int64_t +from hummingbot.strategy.strategy_base cimport StrategyBase + + +cdef class AvellanedaMarketMakingStrategy(StrategyBase): + cdef: + object _market_info + object _minimum_spread + object _order_amount + double _order_refresh_time + double _max_order_age + object _order_refresh_tolerance_pct + double _filled_order_delay + object _inventory_target_base_pct + bint _order_optimization_enabled + bint _add_transaction_costs_to_orders + bint _hb_app_notification + bint _is_debug + + double _cancel_timestamp + double _create_timestamp + object _limit_order_type + bint _all_markets_ready + int _filled_buys_balance + int _filled_sells_balance + double _last_timestamp + double _status_report_interval + int64_t _logging_options + object _last_own_trade_price + int _volatility_sampling_period + double _last_sampling_timestamp + bint _parameters_based_on_spread + int _ticks_to_be_ready + object _min_spread + object _max_spread + object _vol_to_spread_multiplier + object _inventory_risk_aversion + object _kappa + object _gamma + object _eta + object _closing_time + object _time_left + object _q_adjustment_factor + object _reserved_price + object _optimal_spread + object _optimal_bid + object _optimal_ask + object _latest_parameter_calculation_vol + str _debug_csv_path + object _avg_vol + + cdef object c_get_mid_price(self) + cdef object c_create_base_proposal(self) + cdef tuple c_get_adjusted_available_balance(self, list orders) + cdef c_apply_order_price_modifiers(self, object proposal) + cdef c_apply_order_amount_eta_transformation(self, object proposal) + cdef c_apply_budget_constraint(self, object proposal) + cdef c_apply_order_optimization(self, object proposal) + cdef c_apply_add_transaction_costs(self, object proposal) + cdef bint c_is_within_tolerance(self, list current_prices, list proposal_prices) + cdef c_cancel_active_orders(self, object proposal) + cdef c_aged_order_refresh(self) + cdef bint c_to_create_orders(self, object proposal) + cdef c_execute_orders_proposal(self, object proposal) + cdef set_timers(self) + cdef double c_get_spread(self) + cdef c_collect_market_variables(self, double timestamp) + cdef bint c_is_algorithm_ready(self) + cdef c_calculate_reserved_price_and_optimal_spread(self) + cdef object c_calculate_target_inventory(self) + cdef c_recalculate_parameters(self) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx new file mode 100644 index 0000000000..9704ec2032 --- /dev/null +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx @@ -0,0 +1,1022 @@ +from decimal import Decimal +import logging +import pandas as pd +import numpy as np +from typing import ( + List, + Dict, +) +from math import ( + floor, + ceil +) +import time +import datetime +import os +from hummingbot.core.clock cimport Clock +from hummingbot.core.event.events import TradeType +from hummingbot.core.data_type.limit_order cimport LimitOrder +from hummingbot.core.data_type.limit_order import LimitOrder +from hummingbot.core.network_iterator import NetworkStatus +from hummingbot.connector.exchange_base import ExchangeBase +from hummingbot.connector.exchange_base cimport ExchangeBase +from hummingbot.core.event.events import OrderType + +from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple +from hummingbot.strategy.strategy_base import StrategyBase +from hummingbot.client.config.global_config_map import global_config_map + +from .data_types import ( + Proposal, + PriceSize +) +from ..order_tracker cimport OrderTracker +from ..__utils__.trailing_indicators.average_volatility import AverageVolatilityIndicator + + +NaN = float("nan") +s_decimal_zero = Decimal(0) +s_decimal_neg_one = Decimal(-1) +s_decimal_one = Decimal(1) +pmm_logger = None + + +cdef class AvellanedaMarketMakingStrategy(StrategyBase): + OPTION_LOG_CREATE_ORDER = 1 << 3 + OPTION_LOG_MAKER_ORDER_FILLED = 1 << 4 + OPTION_LOG_STATUS_REPORT = 1 << 5 + OPTION_LOG_ALL = 0x7fffffffffffffff + + # These are exchanges where you're expected to expire orders instead of actively cancelling them. + RADAR_RELAY_TYPE_EXCHANGES = {"radar_relay", "bamboo_relay"} + + @classmethod + def logger(cls): + global pmm_logger + if pmm_logger is None: + pmm_logger = logging.getLogger(__name__) + return pmm_logger + + def __init__(self, + market_info: MarketTradingPairTuple, + order_amount: Decimal, + order_refresh_time: float = 30.0, + max_order_age = 1800.0, + order_refresh_tolerance_pct: Decimal = s_decimal_neg_one, + order_optimization_enabled = True, + filled_order_delay: float = 60.0, + inventory_target_base_pct: Decimal = s_decimal_zero, + add_transaction_costs_to_orders: bool = True, + logging_options: int = OPTION_LOG_ALL, + status_report_interval: float = 900, + hb_app_notification: bool = False, + parameters_based_on_spread: bool = True, + min_spread: Decimal = Decimal("0.15"), + max_spread: Decimal = Decimal("2"), + vol_to_spread_multiplier: Decimal = Decimal("1.3"), + inventory_risk_aversion: Decimal = Decimal("0.5"), + order_book_depth_factor: Decimal = Decimal("0.1"), + risk_factor: Decimal = Decimal("0.5"), + order_amount_shape_factor: Decimal = Decimal("0.005"), + closing_time: Decimal = Decimal("1"), + debug_csv_path: str = '', + volatility_buffer_size: int = 30, + is_debug: bool = True, + ): + super().__init__() + self._sb_order_tracker = OrderTracker() + self._market_info = market_info + self._order_amount = order_amount + self._order_optimization_enabled = order_optimization_enabled + self._order_refresh_time = order_refresh_time + self._max_order_age = max_order_age + self._order_refresh_tolerance_pct = order_refresh_tolerance_pct + self._filled_order_delay = filled_order_delay + self._inventory_target_base_pct = inventory_target_base_pct + self._add_transaction_costs_to_orders = add_transaction_costs_to_orders + self._hb_app_notification = hb_app_notification + + self._cancel_timestamp = 0 + self._create_timestamp = 0 + self._limit_order_type = self._market_info.market.get_maker_order_type() + self._all_markets_ready = False + self._filled_buys_balance = 0 + self._filled_sells_balance = 0 + self._logging_options = logging_options + self._last_timestamp = 0 + self._status_report_interval = status_report_interval + self._last_own_trade_price = Decimal('nan') + + self.c_add_markets([market_info.market]) + self._ticks_to_be_ready = volatility_buffer_size + self._parameters_based_on_spread = parameters_based_on_spread + self._min_spread = min_spread + self._max_spread = max_spread + self._vol_to_spread_multiplier = vol_to_spread_multiplier + self._inventory_risk_aversion = inventory_risk_aversion + self._avg_vol = AverageVolatilityIndicator(volatility_buffer_size, 1) + self._last_sampling_timestamp = 0 + self._kappa = order_book_depth_factor + self._gamma = risk_factor + self._eta = order_amount_shape_factor + self._time_left = closing_time + self._closing_time = closing_time + self._latest_parameter_calculation_vol = s_decimal_zero + self._reserved_price = s_decimal_zero + self._optimal_spread = s_decimal_zero + self._optimal_ask = s_decimal_zero + self._optimal_bid = s_decimal_zero + self._debug_csv_path = debug_csv_path + self._is_debug = is_debug + try: + if self._is_debug: + os.unlink(self._debug_csv_path) + except FileNotFoundError: + pass + + def all_markets_ready(self): + return all([market.ready for market in self._sb_markets]) + + @property + def market_info(self) -> MarketTradingPairTuple: + return self._market_info + + @property + def order_refresh_tolerance_pct(self) -> Decimal: + return self._order_refresh_tolerance_pct + + @order_refresh_tolerance_pct.setter + def order_refresh_tolerance_pct(self, value: Decimal): + self._order_refresh_tolerance_pct = value + + @property + def order_amount(self) -> Decimal: + return self._order_amount + + @order_amount.setter + def order_amount(self, value: Decimal): + self._order_amount = value + + @property + def inventory_target_base_pct(self) -> Decimal: + return self._inventory_target_base_pct + + @inventory_target_base_pct.setter + def inventory_target_base_pct(self, value: Decimal): + self._inventory_target_base_pct = value + + @property + def order_optimization_enabled(self) -> bool: + return self._order_optimization_enabled + + @order_optimization_enabled.setter + def order_optimization_enabled(self, value: bool): + self._order_optimization_enabled = value + + @property + def order_refresh_time(self) -> float: + return self._order_refresh_time + + @order_refresh_time.setter + def order_refresh_time(self, value: float): + self._order_refresh_time = value + + @property + def filled_order_delay(self) -> float: + return self._filled_order_delay + + @filled_order_delay.setter + def filled_order_delay(self, value: float): + self._filled_order_delay = value + + @property + def filled_order_delay(self) -> float: + return self._filled_order_delay + + @filled_order_delay.setter + def filled_order_delay(self, value: float): + self._filled_order_delay = value + + @property + def add_transaction_costs_to_orders(self) -> bool: + return self._add_transaction_costs_to_orders + + @add_transaction_costs_to_orders.setter + def add_transaction_costs_to_orders(self, value: bool): + self._add_transaction_costs_to_orders = value + + @property + def base_asset(self): + return self._market_info.base_asset + + @property + def quote_asset(self): + return self._market_info.quote_asset + + @property + def trading_pair(self): + return self._market_info.trading_pair + + def get_price(self) -> float: + return self.get_mid_price() + + def get_last_price(self) -> float: + return self._market_info.get_last_price() + + def get_mid_price(self) -> float: + return self.c_get_mid_price() + + cdef object c_get_mid_price(self): + return self._market_info.get_mid_price() + + @property + def market_info_to_active_orders(self) -> Dict[MarketTradingPairTuple, List[LimitOrder]]: + return self._sb_order_tracker.market_pair_to_active_orders + + @property + def active_orders(self) -> List[LimitOrder]: + if self._market_info not in self.market_info_to_active_orders: + return [] + return self.market_info_to_active_orders[self._market_info] + + @property + def active_buys(self) -> List[LimitOrder]: + return [o for o in self.active_orders if o.is_buy] + + @property + def active_sells(self) -> List[LimitOrder]: + return [o for o in self.active_orders if not o.is_buy] + + @property + def logging_options(self) -> int: + return self._logging_options + + @logging_options.setter + def logging_options(self, int64_t logging_options): + self._logging_options = logging_options + + @property + def order_tracker(self): + return self._sb_order_tracker + + def pure_mm_assets_df(self, to_show_current_pct: bool) -> pd.DataFrame: + market, trading_pair, base_asset, quote_asset = self._market_info + price = self._market_info.get_mid_price() + base_balance = float(market.get_balance(base_asset)) + quote_balance = float(market.get_balance(quote_asset)) + available_base_balance = float(market.get_available_balance(base_asset)) + available_quote_balance = float(market.get_available_balance(quote_asset)) + base_value = base_balance * float(price) + total_in_quote = base_value + quote_balance + base_ratio = base_value / total_in_quote if total_in_quote > 0 else 0 + quote_ratio = quote_balance / total_in_quote if total_in_quote > 0 else 0 + data=[ + ["", base_asset, quote_asset], + ["Total Balance", round(base_balance, 4), round(quote_balance, 4)], + ["Available Balance", round(available_base_balance, 4), round(available_quote_balance, 4)], + [f"Current Value ({quote_asset})", round(base_value, 4), round(quote_balance, 4)] + ] + if to_show_current_pct: + data.append(["Current %", f"{base_ratio:.1%}", f"{quote_ratio:.1%}"]) + df = pd.DataFrame(data=data) + return df + + def active_orders_df(self) -> pd.DataFrame: + market, trading_pair, base_asset, quote_asset = self._market_info + price = self.get_price() + active_orders = self.active_orders + no_sells = len([o for o in active_orders if not o.is_buy and o.client_order_id]) + active_orders.sort(key=lambda x: x.price, reverse=True) + columns = ["Level", "Type", "Price", "Spread", "Amount (Orig)", "Amount (Adj)", "Age"] + data = [] + lvl_buy, lvl_sell = 0, 0 + for idx in range(0, len(active_orders)): + order = active_orders[idx] + spread = 0 if price == 0 else abs(order.price - price)/price + age = "n/a" + # // indicates order is a paper order so 'n/a'. For real orders, calculate age. + if "//" not in order.client_order_id: + age = pd.Timestamp(int(time.time()) - int(order.client_order_id[-16:])/1e6, + unit='s').strftime('%H:%M:%S') + amount_orig = self._order_amount + data.append([ + "", + "buy" if order.is_buy else "sell", + float(order.price), + f"{spread:.2%}", + amount_orig, + float(order.quantity), + age + ]) + + return pd.DataFrame(data=data, columns=columns) + + def market_status_data_frame(self, market_trading_pair_tuples: List[MarketTradingPairTuple]) -> pd.DataFrame: + markets_data = [] + markets_columns = ["Exchange", "Market", "Best Bid", "Best Ask", f"MidPrice"] + markets_columns.append('Reserved Price') + market_books = [(self._market_info.market, self._market_info.trading_pair)] + for market, trading_pair in market_books: + bid_price = market.get_price(trading_pair, False) + ask_price = market.get_price(trading_pair, True) + ref_price = self.get_price() + markets_data.append([ + market.display_name, + trading_pair, + float(bid_price), + float(ask_price), + float(ref_price), + round(self._reserved_price, 5), + ]) + return pd.DataFrame(data=markets_data, columns=markets_columns).replace(np.nan, '', regex=True) + + def format_status(self) -> str: + if not self._all_markets_ready: + return "Market connectors are not ready." + cdef: + list lines = [] + list warning_lines = [] + warning_lines.extend(self.network_warning([self._market_info])) + + markets_df = self.market_status_data_frame([self._market_info]) + lines.extend(["", " Markets:"] + [" " + line for line in markets_df.to_string(index=False).split("\n")]) + + assets_df = self.pure_mm_assets_df(True) + first_col_length = max(*assets_df[0].apply(len)) + df_lines = assets_df.to_string(index=False, header=False, + formatters={0: ("{:<" + str(first_col_length) + "}").format}).split("\n") + lines.extend(["", " Assets:"] + [" " + line for line in df_lines]) + + # See if there are any open orders. + if len(self.active_orders) > 0: + df = self.active_orders_df() + lines.extend(["", " Orders:"] + [" " + line for line in df.to_string(index=False).split("\n")]) + else: + lines.extend(["", " No active maker orders."]) + + volatility_pct = self._avg_vol.current_value / float(self.get_price()) * 100.0 + if all((self._gamma, self._kappa, volatility_pct)): + lines.extend(["", f" Strategy parameters:", + f" risk_factor(\u03B3)= {self._gamma:.5E}", + f" order_book_depth_factor(\u03BA)= {self._kappa:.5E}", + f" volatility= {volatility_pct:.3f}%", + f" time until end of trading cycle= {str(datetime.timedelta(seconds=float(self._time_left)//1e3))}"]) + + warning_lines.extend(self.balance_warning([self._market_info])) + + if len(warning_lines) > 0: + lines.extend(["", "*** WARNINGS ***"] + warning_lines) + + return "\n".join(lines) + + # The following exposed Python functions are meant for unit tests + # --------------------------------------------------------------- + def execute_orders_proposal(self, proposal: Proposal): + return self.c_execute_orders_proposal(proposal) + + def cancel_order(self, order_id: str): + return self.c_cancel_order(self._market_info, order_id) + + # --------------------------------------------------------------- + + cdef c_start(self, Clock clock, double timestamp): + StrategyBase.c_start(self, clock, timestamp) + self._last_timestamp = timestamp + # start tracking any restored limit order + restored_order_ids = self.c_track_restored_orders(self.market_info) + self._time_left = self._closing_time + + cdef c_tick(self, double timestamp): + StrategyBase.c_tick(self, timestamp) + cdef: + int64_t current_tick = (timestamp // self._status_report_interval) + int64_t last_tick = (self._last_timestamp // self._status_report_interval) + bint should_report_warnings = ((current_tick > last_tick) and + (self._logging_options & self.OPTION_LOG_STATUS_REPORT)) + cdef object proposal + try: + if not self._all_markets_ready: + self._all_markets_ready = all([mkt.ready for mkt in self._sb_markets]) + if not self._all_markets_ready: + # Markets not ready yet. Don't do anything. + if should_report_warnings: + self.logger().warning(f"Markets are not ready. No market making trades are permitted.") + return + + if should_report_warnings: + if not all([mkt.network_status is NetworkStatus.CONNECTED for mkt in self._sb_markets]): + self.logger().warning(f"WARNING: Some markets are not connected or are down at the moment. Market " + f"making may be dangerous when markets or networks are unstable.") + + self.c_collect_market_variables(timestamp) + if self.c_is_algorithm_ready(): + # If gamma or kappa are -1 then it's the first time they are calculated. + # Also, if volatility goes beyond the threshold specified, we consider volatility regime has changed + # so parameters need to be recalculated. + if (self._gamma is None) or (self._kappa is None) or \ + (self._parameters_based_on_spread and + self.volatility_diff_from_last_parameter_calculation(self.get_volatility()) > (self._vol_to_spread_multiplier - 1)): + self.c_recalculate_parameters() + self.c_calculate_reserved_price_and_optimal_spread() + + proposal = None + if self._create_timestamp <= self._current_timestamp: + # 1. Create base order proposals + proposal = self.c_create_base_proposal() + # 2. Apply functions that modify orders amount + self.c_apply_order_amount_eta_transformation(proposal) + # 3. Apply functions that modify orders price + self.c_apply_order_price_modifiers(proposal) + # 4. Apply budget constraint, i.e. can't buy/sell more than what you have. + self.c_apply_budget_constraint(proposal) + + self.c_cancel_active_orders(proposal) + if self._is_debug: + self.dump_debug_variables() + refresh_proposal = self.c_aged_order_refresh() + # Firstly restore cancelled aged order + if refresh_proposal is not None: + self.c_execute_orders_proposal(refresh_proposal) + if self.c_to_create_orders(proposal): + self.c_execute_orders_proposal(proposal) + else: + self._ticks_to_be_ready-=1 + if self._ticks_to_be_ready % 5 == 0: + self.logger().info(f"Calculating volatility... {self._ticks_to_be_ready} seconds to start trading") + finally: + self._last_timestamp = timestamp + + cdef c_collect_market_variables(self, double timestamp): + market, trading_pair, base_asset, quote_asset = self._market_info + self._last_sampling_timestamp = timestamp + self._time_left = max(self._time_left - Decimal(timestamp - self._last_timestamp) * 1000, 0) + price = self.get_price() + self._avg_vol.add_sample(price) + # Calculate adjustment factor to have 0.01% of inventory resolution + base_balance = market.get_balance(base_asset) + quote_balance = market.get_balance(quote_asset) + inventory_in_base = quote_balance / price + base_balance + self._q_adjustment_factor = Decimal( + "1e5") / inventory_in_base + if self._time_left == 0: + # Re-cycle algorithm + self._time_left = self._closing_time + if self._parameters_based_on_spread: + self.c_recalculate_parameters() + self.logger().info("Recycling algorithm time left and parameters if needed.") + + def volatility_diff_from_last_parameter_calculation(self, current_vol): + if self._latest_parameter_calculation_vol == 0: + return s_decimal_zero + return abs(self._latest_parameter_calculation_vol - Decimal(str(current_vol))) / self._latest_parameter_calculation_vol + + cdef double c_get_spread(self): + cdef: + ExchangeBase market = self._market_info.market + str trading_pair = self._market_info.trading_pair + + return market.c_get_price(trading_pair, True) - market.c_get_price(trading_pair, False) + + def get_volatility(self): + vol = Decimal(str(self._avg_vol.current_value)) + if vol == s_decimal_zero: + if self._latest_parameter_calculation_vol != s_decimal_zero: + vol = Decimal(str(self._latest_parameter_calculation_vol)) + else: + # Default value at start time if price has no activity + vol = Decimal(str(self.c_get_spread()/2)) + return vol + + cdef c_calculate_reserved_price_and_optimal_spread(self): + cdef: + ExchangeBase market = self._market_info.market + + time_left_fraction = Decimal(str(self._time_left / self._closing_time)) + + price = self.get_price() + q = (market.get_balance(self.base_asset) - Decimal(str(self.c_calculate_target_inventory()))) * self._q_adjustment_factor + vol = self.get_volatility() + mid_price_variance = vol ** 2 + + self._reserved_price = price - (q * self._gamma * mid_price_variance * time_left_fraction) + self._optimal_spread = self._gamma * mid_price_variance * time_left_fraction + 2 * Decimal( + 1 + self._gamma / self._kappa).ln() / self._gamma + + if self._parameters_based_on_spread: + min_limit_bid = min(price * (1 - self._max_spread), price - self._vol_to_spread_multiplier * vol) + max_limit_bid = price * (1 - self._min_spread) + min_limit_ask = price * (1 + self._min_spread) + max_limit_ask = max(price * (1 + self._max_spread), price + self._vol_to_spread_multiplier * vol) + else: + min_limit_bid = s_decimal_zero + max_limit_bid = min_limit_ask = price + max_limit_ask = Decimal("Inf") + + self._optimal_ask = min(max(self._reserved_price + self._optimal_spread / 2, + min_limit_ask), + max_limit_ask) + self._optimal_bid = min(max(self._reserved_price - self._optimal_spread / 2, + min_limit_bid), + max_limit_bid) + # This is not what the algorithm will use as proposed bid and ask. This is just the raw output. + # Optimal bid and optimal ask prices will be used + if self._is_debug: + self.logger().info(f"bid={(price-(self._reserved_price - self._optimal_spread / 2)) / price * 100:.4f}% | " + f"ask={((self._reserved_price + self._optimal_spread / 2) - price) / price * 100:.4f}% | " + f"q={q/self._q_adjustment_factor:.4f} | " + f"vol={vol:.4f}") + + cdef object c_calculate_target_inventory(self): + cdef: + ExchangeBase market = self._market_info.market + str trading_pair = self._market_info.trading_pair + str base_asset = self._market_info.base_asset + str quote_asset = self._market_info.quote_asset + object mid_price + object base_value + object inventory_value + object target_inventory_value + + price = self.get_price() + base_asset_amount = market.get_balance(base_asset) + quote_asset_amount = market.get_balance(quote_asset) + base_value = base_asset_amount * price + inventory_value = base_value + quote_asset_amount + target_inventory_value = inventory_value * self._inventory_target_base_pct + return market.c_quantize_order_amount(trading_pair, Decimal(str(target_inventory_value / price))) + + cdef c_recalculate_parameters(self): + cdef: + ExchangeBase market = self._market_info.market + + q = (market.get_balance(self.base_asset) - self.c_calculate_target_inventory()) * self._q_adjustment_factor + vol = self.get_volatility() + price=self.get_price() + + if q != 0: + min_spread = self._min_spread * price + max_spread = self._max_spread * price + + # GAMMA + # If q or vol are close to 0, gamma will -> Inf. Is this desirable? + max_possible_gamma = min( + (max_spread - min_spread) / (2 * abs(q) * (vol ** 2)), + (max_spread * (2-self._inventory_risk_aversion) / + self._inventory_risk_aversion + min_spread) / (vol ** 2)) + self._gamma = self._inventory_risk_aversion * max_possible_gamma + + # KAPPA + # Want the maximum possible spread but with restrictions to avoid negative kappa or division by 0 + max_spread_around_reserved_price = max_spread * (2-self._inventory_risk_aversion) + min_spread * self._inventory_risk_aversion + if max_spread_around_reserved_price <= self._gamma * (vol ** 2): + self._kappa = Decimal('1e100') # Cap to kappa -> Infinity + else: + self._kappa = self._gamma / (Decimal.exp((max_spread_around_reserved_price * self._gamma - (vol * self._gamma) **2) / 2) - 1) + + # ETA + + q_where_to_decay_order_amount = self.c_calculate_target_inventory() * (1 - self._inventory_risk_aversion) + self._eta = s_decimal_one + if q_where_to_decay_order_amount != s_decimal_zero: + self._eta = self._eta / q_where_to_decay_order_amount + + self._latest_parameter_calculation_vol = vol + + cdef bint c_is_algorithm_ready(self): + return self._avg_vol.is_sampling_buffer_full + + cdef object c_create_base_proposal(self): + cdef: + ExchangeBase market = self._market_info.market + list buys = [] + list sells = [] + + price = market.c_quantize_order_price(self.trading_pair, Decimal(str(self._optimal_bid))) + size = market.c_quantize_order_amount(self.trading_pair, self._order_amount) + if size>0: + buys.append(PriceSize(price, size)) + + price = market.c_quantize_order_price(self.trading_pair, Decimal(str(self._optimal_ask))) + size = market.c_quantize_order_amount(self.trading_pair, self._order_amount) + if size>0: + sells.append(PriceSize(price, size)) + + return Proposal(buys, sells) + + cdef tuple c_get_adjusted_available_balance(self, list orders): + """ + Calculates the available balance, plus the amount attributed to orders. + :return: (base amount, quote amount) in Decimal + """ + cdef: + ExchangeBase market = self._market_info.market + object base_balance = market.c_get_available_balance(self.base_asset) + object quote_balance = market.c_get_available_balance(self.quote_asset) + + for order in orders: + if order.is_buy: + quote_balance += order.quantity * order.price + else: + base_balance += order.quantity + + return base_balance, quote_balance + + cdef c_apply_order_price_modifiers(self, object proposal): + if self._order_optimization_enabled: + self.c_apply_order_optimization(proposal) + + if self._add_transaction_costs_to_orders: + self.c_apply_add_transaction_costs(proposal) + + cdef c_apply_budget_constraint(self, object proposal): + cdef: + ExchangeBase market = self._market_info.market + object quote_size + object base_size + object adjusted_amount + + base_balance, quote_balance = self.c_get_adjusted_available_balance(self.active_orders) + + for buy in proposal.buys: + buy_fee = market.c_get_fee(self.base_asset, self.quote_asset, OrderType.LIMIT, TradeType.BUY, + buy.size, buy.price) + quote_size = buy.size * buy.price * (Decimal(1) + buy_fee.percent) + + # Adjust buy order size to use remaining balance if less than the order amount + if quote_balance < quote_size: + adjusted_amount = quote_balance / (buy.price * (Decimal("1") + buy_fee.percent)) + adjusted_amount = market.c_quantize_order_amount(self.trading_pair, adjusted_amount) + buy.size = adjusted_amount + quote_balance = s_decimal_zero + elif quote_balance == s_decimal_zero: + buy.size = s_decimal_zero + else: + quote_balance -= quote_size + + proposal.buys = [o for o in proposal.buys if o.size > 0] + + for sell in proposal.sells: + base_size = sell.size + + # Adjust sell order size to use remaining balance if less than the order amount + if base_balance < base_size: + adjusted_amount = market.c_quantize_order_amount(self.trading_pair, base_balance) + sell.size = adjusted_amount + base_balance = s_decimal_zero + elif base_balance == s_decimal_zero: + sell.size = s_decimal_zero + else: + base_balance -= base_size + + proposal.sells = [o for o in proposal.sells if o.size > 0] + + # Compare the market price with the top bid and top ask price + cdef c_apply_order_optimization(self, object proposal): + cdef: + ExchangeBase market = self._market_info.market + object own_buy_size = s_decimal_zero + object own_sell_size = s_decimal_zero + object best_order_spread + + for order in self.active_orders: + if order.is_buy: + own_buy_size = order.quantity + else: + own_sell_size = order.quantity + + if len(proposal.buys) > 0: + # Get the top bid price in the market using order_optimization_depth and your buy order volume + top_bid_price = self._market_info.get_price_for_volume( + False, own_buy_size).result_price + price_quantum = market.c_get_order_price_quantum( + self.trading_pair, + top_bid_price + ) + # Get the price above the top bid + price_above_bid = (ceil(top_bid_price / price_quantum) + 1) * price_quantum + + # If the price_above_bid is lower than the price suggested by the top pricing proposal, + # lower the price and from there apply the best_order_spread to each order in the next levels + proposal.buys = sorted(proposal.buys, key = lambda p: p.price, reverse = True) + lower_buy_price = min(proposal.buys[0].price, price_above_bid) + for i, proposed in enumerate(proposal.buys): + proposal.buys[i].price = market.c_quantize_order_price(self.trading_pair, lower_buy_price) + + if len(proposal.sells) > 0: + # Get the top ask price in the market using order_optimization_depth and your sell order volume + top_ask_price = self._market_info.get_price_for_volume( + True, own_sell_size).result_price + price_quantum = market.c_get_order_price_quantum( + self.trading_pair, + top_ask_price + ) + # Get the price below the top ask + price_below_ask = (floor(top_ask_price / price_quantum) - 1) * price_quantum + + # If the price_below_ask is higher than the price suggested by the pricing proposal, + # increase your price and from there apply the best_order_spread to each order in the next levels + proposal.sells = sorted(proposal.sells, key = lambda p: p.price) + higher_sell_price = max(proposal.sells[0].price, price_below_ask) + for i, proposed in enumerate(proposal.sells): + proposal.sells[i].price = market.c_quantize_order_price(self.trading_pair, higher_sell_price) + + cdef c_apply_order_amount_eta_transformation(self, object proposal): + cdef: + ExchangeBase market = self._market_info.market + str trading_pair = self._market_info.trading_pair + + # eta parameter is described in the paper as the shape parameter for having exponentially decreasing order amount + # for orders that go against inventory target (i.e. Want to buy when excess inventory or sell when deficit inventory) + q = market.get_balance(self.base_asset) - self.c_calculate_target_inventory() + if len(proposal.buys) > 0: + if q > 0: + for i, proposed in enumerate(proposal.buys): + + proposal.buys[i].size = market.c_quantize_order_amount(trading_pair, proposal.buys[i].size * Decimal.exp(-self._eta * q)) + proposal.buys = [o for o in proposal.buys if o.size > 0] + + if len(proposal.sells) > 0: + if q < 0: + for i, proposed in enumerate(proposal.sells): + proposal.sells[i].size = market.c_quantize_order_amount(trading_pair, proposal.sells[i].size * Decimal.exp(self._eta * q)) + proposal.sells = [o for o in proposal.sells if o.size > 0] + + cdef object c_apply_add_transaction_costs(self, object proposal): + cdef: + ExchangeBase market = self._market_info.market + for buy in proposal.buys: + fee = market.c_get_fee(self.base_asset, self.quote_asset, + self._limit_order_type, TradeType.BUY, buy.size, buy.price) + price = buy.price * (Decimal(1) - fee.percent) + buy.price = market.c_quantize_order_price(self.trading_pair, price) + for sell in proposal.sells: + fee = market.c_get_fee(self.base_asset, self.quote_asset, + self._limit_order_type, TradeType.SELL, sell.size, sell.price) + price = sell.price * (Decimal(1) + fee.percent) + sell.price = market.c_quantize_order_price(self.trading_pair, price) + + cdef c_did_fill_order(self, object order_filled_event): + cdef: + str order_id = order_filled_event.order_id + object market_info = self._sb_order_tracker.c_get_shadow_market_pair_from_order_id(order_id) + tuple order_fill_record + + if market_info is not None: + limit_order_record = self._sb_order_tracker.c_get_shadow_limit_order(order_id) + order_fill_record = (limit_order_record, order_filled_event) + + if order_filled_event.trade_type is TradeType.BUY: + if self._logging_options & self.OPTION_LOG_MAKER_ORDER_FILLED: + self.log_with_clock( + logging.INFO, + f"({market_info.trading_pair}) Maker buy order of " + f"{order_filled_event.amount} {market_info.base_asset} filled." + ) + else: + if self._logging_options & self.OPTION_LOG_MAKER_ORDER_FILLED: + self.log_with_clock( + logging.INFO, + f"({market_info.trading_pair}) Maker sell order of " + f"{order_filled_event.amount} {market_info.base_asset} filled." + ) + + cdef c_did_complete_buy_order(self, object order_completed_event): + cdef: + str order_id = order_completed_event.order_id + limit_order_record = self._sb_order_tracker.c_get_limit_order(self._market_info, order_id) + if limit_order_record is None: + return + active_sell_ids = [x.client_order_id for x in self.active_orders if not x.is_buy] + + # delay order creation by filled_order_delay (in seconds) + self._create_timestamp = self._current_timestamp + self._filled_order_delay + self._cancel_timestamp = min(self._cancel_timestamp, self._create_timestamp) + + self._filled_buys_balance += 1 + self._last_own_trade_price = limit_order_record.price + + self.log_with_clock( + logging.INFO, + f"({self.trading_pair}) Maker buy order {order_id} " + f"({limit_order_record.quantity} {limit_order_record.base_currency} @ " + f"{limit_order_record.price} {limit_order_record.quote_currency}) has been completely filled." + ) + self.notify_hb_app( + f"Maker BUY order {limit_order_record.quantity} {limit_order_record.base_currency} @ " + f"{limit_order_record.price} {limit_order_record.quote_currency} is filled." + ) + + cdef c_did_complete_sell_order(self, object order_completed_event): + cdef: + str order_id = order_completed_event.order_id + LimitOrder limit_order_record = self._sb_order_tracker.c_get_limit_order(self._market_info, order_id) + if limit_order_record is None: + return + active_buy_ids = [x.client_order_id for x in self.active_orders if x.is_buy] + + # delay order creation by filled_order_delay (in seconds) + self._create_timestamp = self._current_timestamp + self._filled_order_delay + self._cancel_timestamp = min(self._cancel_timestamp, self._create_timestamp) + + self._filled_sells_balance += 1 + self._last_own_trade_price = limit_order_record.price + + self.log_with_clock( + logging.INFO, + f"({self.trading_pair}) Maker sell order {order_id} " + f"({limit_order_record.quantity} {limit_order_record.base_currency} @ " + f"{limit_order_record.price} {limit_order_record.quote_currency}) has been completely filled." + ) + self.notify_hb_app( + f"Maker SELL order {limit_order_record.quantity} {limit_order_record.base_currency} @ " + f"{limit_order_record.price} {limit_order_record.quote_currency} is filled." + ) + + cdef bint c_is_within_tolerance(self, list current_prices, list proposal_prices): + if len(current_prices) != len(proposal_prices): + return False + current_prices = sorted(current_prices) + proposal_prices = sorted(proposal_prices) + for current, proposal in zip(current_prices, proposal_prices): + # if spread diff is more than the tolerance or order quantities are different, return false. + if abs(proposal - current)/current > self._order_refresh_tolerance_pct: + return False + return True + + # Cancel active orders + # Return value: whether order cancellation is deferred. + cdef c_cancel_active_orders(self, object proposal): + if self._cancel_timestamp > self._current_timestamp: + return + if not global_config_map.get("0x_active_cancels").value: + if ((self._market_info.market.name in self.RADAR_RELAY_TYPE_EXCHANGES) or + (self._market_info.market.name == "bamboo_relay" and not self._market_info.market.use_coordinator)): + return + + cdef: + list active_orders = self.active_orders + list active_buy_prices = [] + list active_sells = [] + bint to_defer_canceling = False + if len(active_orders) == 0: + return + if proposal is not None: + active_buy_prices = [Decimal(str(o.price)) for o in active_orders if o.is_buy] + active_sell_prices = [Decimal(str(o.price)) for o in active_orders if not o.is_buy] + proposal_buys = [buy.price for buy in proposal.buys] + proposal_sells = [sell.price for sell in proposal.sells] + if self.c_is_within_tolerance(active_buy_prices, proposal_buys) and \ + self.c_is_within_tolerance(active_sell_prices, proposal_sells): + to_defer_canceling = True + + if not to_defer_canceling: + for order in active_orders: + self.c_cancel_order(self._market_info, order.client_order_id) + else: + self.set_timers() + + # Refresh all active order that are older that the _max_order_age + cdef c_aged_order_refresh(self): + cdef: + list active_orders = self.active_orders + list buys = [] + list sells = [] + + for order in active_orders: + age = 0 if "//" in order.client_order_id else \ + int(int(time.time()) - int(order.client_order_id[-16:])/1e6) + + # To prevent duplicating orders due to delay in receiving cancel response + refresh_check = [o for o in active_orders if o.price == order.price + and o.quantity == order.quantity] + if len(refresh_check) > 1: + continue + + if age >= self._max_order_age: + if order.is_buy: + buys.append(PriceSize(order.price, order.quantity)) + else: + sells.append(PriceSize(order.price, order.quantity)) + self.logger().info(f"Refreshing {'Buy' if order.is_buy else 'Sell'} order with ID - " + f"{order.client_order_id} because it reached maximum order age of " + f"{self._max_order_age} seconds.") + self.c_cancel_order(self._market_info, order.client_order_id) + return Proposal(buys, sells) + + cdef bint c_to_create_orders(self, object proposal): + return self._create_timestamp < self._current_timestamp and \ + proposal is not None + + cdef c_execute_orders_proposal(self, object proposal): + cdef: + double expiration_seconds = (self._order_refresh_time + if ((self._market_info.market.name in self.RADAR_RELAY_TYPE_EXCHANGES) or + (self._market_info.market.name == "bamboo_relay" and + not self._market_info.market.use_coordinator)) + else NaN) + str bid_order_id, ask_order_id + bint orders_created = False + + if len(proposal.buys) > 0: + if self._logging_options & self.OPTION_LOG_CREATE_ORDER: + price_quote_str = [f"{buy.size.normalize()} {self.base_asset}, " + f"{buy.price.normalize()} {self.quote_asset}" + for buy in proposal.buys] + self.logger().info( + f"({self.trading_pair}) Creating {len(proposal.buys)} bid orders " + f"at (Size, Price): {price_quote_str}" + ) + for buy in proposal.buys: + bid_order_id = self.c_buy_with_specific_market( + self._market_info, + buy.size, + order_type=self._limit_order_type, + price=buy.price, + expiration_seconds=expiration_seconds + ) + orders_created = True + if len(proposal.sells) > 0: + if self._logging_options & self.OPTION_LOG_CREATE_ORDER: + price_quote_str = [f"{sell.size.normalize()} {self.base_asset}, " + f"{sell.price.normalize()} {self.quote_asset}" + for sell in proposal.sells] + self.logger().info( + f"({self.trading_pair}) Creating {len(proposal.sells)} ask " + f"orders at (Size, Price): {price_quote_str}" + ) + for sell in proposal.sells: + ask_order_id = self.c_sell_with_specific_market( + self._market_info, + sell.size, + order_type=self._limit_order_type, + price=sell.price, + expiration_seconds=expiration_seconds + ) + orders_created = True + if orders_created: + self.set_timers() + + cdef set_timers(self): + cdef double next_cycle = self._current_timestamp + self._order_refresh_time + if self._create_timestamp <= self._current_timestamp: + self._create_timestamp = next_cycle + if self._cancel_timestamp <= self._current_timestamp: + self._cancel_timestamp = min(self._create_timestamp, next_cycle) + + def notify_hb_app(self, msg: str): + if self._hb_app_notification: + from hummingbot.client.hummingbot_application import HummingbotApplication + HummingbotApplication.main_application()._notify(msg) + + def dump_debug_variables(self): + market = self._market_info.market + mid_price = self.get_price() + spread = Decimal(str(self.c_get_spread())) + + best_ask = mid_price + spread / 2 + new_ask = self._reserved_price + self._optimal_spread / 2 + best_bid = mid_price - spread / 2 + new_bid = self._reserved_price - self._optimal_spread / 2 + if not os.path.exists(self._debug_csv_path): + df_header = pd.DataFrame([('mid_price', + 'spread', + 'reserved_price', + 'optimal_spread', + 'optimal_bid', + 'optimal_ask', + 'optimal_bid_to_mid_%', + 'optimal_ask_to_mid_%', + 'current_inv', + 'target_inv', + 'time_left_fraction', + 'mid_price std_dev', + 'gamma', + 'kappa', + 'eta', + 'current_vol_to_calculation_vol', + 'inventory_target_pct', + 'min_spread', + 'max_spread', + 'vol_to_spread_multiplier')]) + df_header.to_csv(self._debug_csv_path, mode='a', header=False, index=False) + df = pd.DataFrame([(mid_price, + spread, + self._reserved_price, + self._optimal_spread, + self._optimal_bid, + self._optimal_ask, + (mid_price - (self._reserved_price - self._optimal_spread / 2)) / mid_price, + ((self._reserved_price + self._optimal_spread / 2) - mid_price) / mid_price, + market.get_balance(self.base_asset), + self.c_calculate_target_inventory(), + self._time_left / self._closing_time, + self._avg_vol.current_value, + self._gamma, + self._kappa, + self._eta, + self.volatility_diff_from_last_parameter_calculation(self.get_volatility()), + self.inventory_target_base_pct, + self._min_spread, + self._max_spread, + self._vol_to_spread_multiplier)]) + df.to_csv(self._debug_csv_path, mode='a', header=False, index=False) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map.py new file mode 100644 index 0000000000..34a6617163 --- /dev/null +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map.py @@ -0,0 +1,243 @@ +from decimal import Decimal + +from hummingbot.client.config.config_var import ConfigVar +from hummingbot.client.config.config_validators import ( + validate_exchange, + validate_market_trading_pair, + validate_bool, + validate_decimal, +) +from hummingbot.client.settings import ( + required_exchanges, + EXAMPLE_PAIRS, +) +from hummingbot.client.config.global_config_map import ( + using_bamboo_coordinator_mode, + using_exchange +) +from hummingbot.client.config.config_helpers import ( + minimum_order_amount, +) +from typing import Optional + + +def maker_trading_pair_prompt(): + exchange = avellaneda_market_making_config_map.get("exchange").value + example = EXAMPLE_PAIRS.get(exchange) + return "Enter the token trading pair you would like to trade on %s%s >>> " \ + % (exchange, f" (e.g. {example})" if example else "") + + +# strategy specific validators +def validate_exchange_trading_pair(value: str) -> Optional[str]: + exchange = avellaneda_market_making_config_map.get("exchange").value + return validate_market_trading_pair(exchange, value) + + +def validate_max_spread(value: str) -> Optional[str]: + validate_decimal(value, 0, 100, inclusive=False) + if avellaneda_market_making_config_map["min_spread"].value is not None: + min_spread = Decimal(avellaneda_market_making_config_map["min_spread"].value) + max_spread = Decimal(value) + if min_spread >= max_spread: + return f"Max spread cannot be lesser or equal to min spread {max_spread}%<={min_spread}%" + + +def onvalidated_min_spread(value: str): + # If entered valid min_spread, max_spread is invalidated so user sets it up again + avellaneda_market_making_config_map["max_spread"].value = None + + +async def order_amount_prompt() -> str: + exchange = avellaneda_market_making_config_map["exchange"].value + trading_pair = avellaneda_market_making_config_map["market"].value + base_asset, quote_asset = trading_pair.split("-") + min_amount = await minimum_order_amount(exchange, trading_pair) + return f"What is the amount of {base_asset} per order? (minimum {min_amount}) >>> " + + +async def validate_order_amount(value: str) -> Optional[str]: + try: + exchange = avellaneda_market_making_config_map["exchange"].value + trading_pair = avellaneda_market_making_config_map["market"].value + min_amount = await minimum_order_amount(exchange, trading_pair) + if Decimal(value) < min_amount: + return f"Order amount must be at least {min_amount}." + except Exception: + return "Invalid order amount." + + +def on_validated_price_source_exchange(value: str): + if value is None: + avellaneda_market_making_config_map["price_source_market"].value = None + + +def exchange_on_validated(value: str): + required_exchanges.append(value) + + +def on_validated_parameters_based_on_spread(value: str): + if value == 'True': + avellaneda_market_making_config_map.get("risk_factor").value = None + avellaneda_market_making_config_map.get("order_book_depth_factor").value = None + avellaneda_market_making_config_map.get("order_amount_shape_factor").value = None + else: + avellaneda_market_making_config_map.get("max_spread").value = None + avellaneda_market_making_config_map.get("min_spread").value = None + avellaneda_market_making_config_map.get("vol_to_spread_multiplier").value = None + avellaneda_market_making_config_map.get("inventory_risk_aversion").value = None + + +avellaneda_market_making_config_map = { + "strategy": + ConfigVar(key="strategy", + prompt=None, + default="avellaneda_market_making"), + "exchange": + ConfigVar(key="exchange", + prompt="Enter your maker spot connector >>> ", + validator=validate_exchange, + on_validated=exchange_on_validated, + prompt_on_new=True), + "market": + ConfigVar(key="market", + prompt=maker_trading_pair_prompt, + validator=validate_exchange_trading_pair, + prompt_on_new=True), + "order_amount": + ConfigVar(key="order_amount", + prompt=order_amount_prompt, + type_str="decimal", + validator=validate_order_amount, + prompt_on_new=True), + "order_optimization_enabled": + ConfigVar(key="order_optimization_enabled", + prompt="Do you want to enable best bid ask jumping? (Yes/No) >>> ", + type_str="bool", + default=True, + validator=validate_bool), + "parameters_based_on_spread": + ConfigVar(key="parameters_based_on_spread", + prompt="Do you want to automate Avellaneda-Stoikov parameters based on min/max spread? >>> ", + type_str="bool", + validator=validate_bool, + on_validated=on_validated_parameters_based_on_spread, + default=True, + prompt_on_new=True), + "min_spread": + ConfigVar(key="min_spread", + prompt="Enter the minimum spread allowed from mid-price in percentage " + "(Enter 1 to indicate 1%) >>> ", + type_str="decimal", + required_if=lambda: avellaneda_market_making_config_map.get("parameters_based_on_spread").value, + validator=lambda v: validate_decimal(v, 0, 100, inclusive=False), + prompt_on_new=True, + on_validated=onvalidated_min_spread), + "max_spread": + ConfigVar(key="max_spread", + prompt="Enter the maximum spread allowed from mid-price in percentage " + "(Enter 1 to indicate 1%) >>> ", + type_str="decimal", + required_if=lambda: avellaneda_market_making_config_map.get("parameters_based_on_spread").value, + validator=lambda v: validate_max_spread(v), + prompt_on_new=True), + "vol_to_spread_multiplier": + ConfigVar(key="vol_to_spread_multiplier", + prompt="Enter the Volatility threshold multiplier (Should be greater than 1.0): " + "(If market volatility multiplied by this value is above the maximum spread, it will increase the maximum spread value) >>>", + type_str="decimal", + required_if=lambda: avellaneda_market_making_config_map.get("parameters_based_on_spread").value, + validator=lambda v: validate_decimal(v, 1, 10, inclusive=False), + prompt_on_new=True), + "inventory_risk_aversion": + ConfigVar(key="inventory_risk_aversion", + prompt="Enter Inventory risk aversion between 0 and 1: (For values close to 0.999 spreads will be more " + "skewed to meet the inventory target, while close to 0.001 spreads will be close to symmetrical, " + "increasing profitability but also increasing inventory risk)>>>", + type_str="decimal", + required_if=lambda: avellaneda_market_making_config_map.get("parameters_based_on_spread").value, + validator=lambda v: validate_decimal(v, 0, 1, inclusive=False), + prompt_on_new=True), + "order_book_depth_factor": + ConfigVar(key="order_book_depth_factor", + printable_key="order_book_depth_factor(\u03BA)", + prompt="Enter order book depth factor (\u03BA) >>> ", + type_str="decimal", + required_if=lambda: not avellaneda_market_making_config_map.get("parameters_based_on_spread").value, + validator=lambda v: validate_decimal(v, 0, 1e10, inclusive=False), + prompt_on_new=True), + "risk_factor": + ConfigVar(key="risk_factor", + printable_key="risk_factor(\u03B3)", + prompt="Enter risk factor (\u03B3) >>> ", + type_str="decimal", + required_if=lambda: not avellaneda_market_making_config_map.get("parameters_based_on_spread").value, + validator=lambda v: validate_decimal(v, 0, 1e10, inclusive=False), + prompt_on_new=True), + "order_amount_shape_factor": + ConfigVar(key="order_amount_shape_factor", + printable_key="order_amount_shape_factor(\u03B7)", + prompt="Enter order amount shape factor (\u03B7) >>> ", + type_str="decimal", + required_if=lambda: not avellaneda_market_making_config_map.get("parameters_based_on_spread").value, + validator=lambda v: validate_decimal(v, 0, 1, inclusive=True), + prompt_on_new=True), + "closing_time": + ConfigVar(key="closing_time", + prompt="Enter operational closing time (T). (How long will each trading cycle last " + "in days or fractions of day) >>> ", + type_str="decimal", + validator=lambda v: validate_decimal(v, 0, 10, inclusive=False), + default=Decimal("0.041666667")), + "order_refresh_time": + ConfigVar(key="order_refresh_time", + prompt="How often do you want to cancel and replace bids and asks " + "(in seconds)? >>> ", + required_if=lambda: not (using_exchange("radar_relay")() or + (using_exchange("bamboo_relay")() and not using_bamboo_coordinator_mode())), + type_str="float", + validator=lambda v: validate_decimal(v, 0, inclusive=False), + prompt_on_new=True), + "max_order_age": + ConfigVar(key="max_order_age", + prompt="How long do you want to cancel and replace bids and asks " + "with the same price (in seconds)? >>> ", + required_if=lambda: not (using_exchange("radar_relay")() or + (using_exchange("bamboo_relay")() and not using_bamboo_coordinator_mode())), + type_str="float", + default=Decimal("1800"), + validator=lambda v: validate_decimal(v, 0, inclusive=False)), + "order_refresh_tolerance_pct": + ConfigVar(key="order_refresh_tolerance_pct", + prompt="Enter the percent change in price needed to refresh orders at each cycle " + "(Enter 1 to indicate 1%) >>> ", + type_str="decimal", + default=Decimal("0"), + validator=lambda v: validate_decimal(v, -10, 10, inclusive=True)), + "filled_order_delay": + ConfigVar(key="filled_order_delay", + prompt="How long do you want to wait before placing the next order " + "if your order gets filled (in seconds)? >>> ", + type_str="float", + validator=lambda v: validate_decimal(v, min_value=0, inclusive=False), + default=60), + "inventory_target_base_pct": + ConfigVar(key="inventory_target_base_pct", + prompt="What is the inventory target for the base asset? Enter 50 for 50% >>> ", + type_str="decimal", + validator=lambda v: validate_decimal(v, 0, 100), + prompt_on_new=True, + default=Decimal("50")), + "add_transaction_costs": + ConfigVar(key="add_transaction_costs", + prompt="Do you want to add transaction costs automatically to order prices? (Yes/No) >>> ", + type_str="bool", + default=False, + validator=validate_bool), + "volatility_buffer_size": + ConfigVar(key="volatility_buffer_size", + prompt="Enter amount of ticks that will be stored to calculate volatility>>> ", + type_str="int", + validator=lambda v: validate_decimal(v, 5, 600), + default=60), +} diff --git a/hummingbot/strategy/avellaneda_market_making/data_types.py b/hummingbot/strategy/avellaneda_market_making/data_types.py new file mode 100644 index 0000000000..ce86cd6091 --- /dev/null +++ b/hummingbot/strategy/avellaneda_market_making/data_types.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +from typing import ( + NamedTuple, + List +) +from decimal import Decimal +from hummingbot.core.event.events import OrderType + +ORDER_PROPOSAL_ACTION_CREATE_ORDERS = 1 +ORDER_PROPOSAL_ACTION_CANCEL_ORDERS = 1 << 1 + + +class OrdersProposal(NamedTuple): + actions: int + buy_order_type: OrderType + buy_order_prices: List[Decimal] + buy_order_sizes: List[Decimal] + sell_order_type: OrderType + sell_order_prices: List[Decimal] + sell_order_sizes: List[Decimal] + cancel_order_ids: List[str] + + +class PricingProposal(NamedTuple): + buy_order_prices: List[Decimal] + sell_order_prices: List[Decimal] + + +class SizingProposal(NamedTuple): + buy_order_sizes: List[Decimal] + sell_order_sizes: List[Decimal] + + +class PriceSize: + def __init__(self, price: Decimal, size: Decimal): + self.price: Decimal = price + self.size: Decimal = size + + def __repr__(self): + return f"[ p: {self.price} s: {self.size} ]" + + +class Proposal: + def __init__(self, buys: List[PriceSize], sells: List[PriceSize]): + self.buys: List[PriceSize] = buys + self.sells: List[PriceSize] = sells + + def __repr__(self): + return f"{len(self.buys)} buys: {', '.join([str(o) for o in self.buys])} " \ + f"{len(self.sells)} sells: {', '.join([str(o) for o in self.sells])}" diff --git a/hummingbot/strategy/avellaneda_market_making/start.py b/hummingbot/strategy/avellaneda_market_making/start.py new file mode 100644 index 0000000000..2d12fe8a39 --- /dev/null +++ b/hummingbot/strategy/avellaneda_market_making/start.py @@ -0,0 +1,85 @@ +from typing import ( + List, + Tuple, +) + +from hummingbot import data_path +import os.path +from hummingbot.client.hummingbot_application import HummingbotApplication +from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple +from hummingbot.strategy.avellaneda_market_making import ( + AvellanedaMarketMakingStrategy, +) +from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map import avellaneda_market_making_config_map as c_map +from decimal import Decimal +import pandas as pd + + +def start(self): + try: + order_amount = c_map.get("order_amount").value + order_optimization_enabled = c_map.get("order_optimization_enabled").value + order_refresh_time = c_map.get("order_refresh_time").value + exchange = c_map.get("exchange").value.lower() + raw_trading_pair = c_map.get("market").value + inventory_target_base_pct = 0 if c_map.get("inventory_target_base_pct").value is None else \ + c_map.get("inventory_target_base_pct").value / Decimal('100') + filled_order_delay = c_map.get("filled_order_delay").value + order_refresh_tolerance_pct = c_map.get("order_refresh_tolerance_pct").value / Decimal('100') + add_transaction_costs_to_orders = c_map.get("add_transaction_costs").value + + trading_pair: str = raw_trading_pair + maker_assets: Tuple[str, str] = self._initialize_market_assets(exchange, [trading_pair])[0] + market_names: List[Tuple[str, List[str]]] = [(exchange, [trading_pair])] + self._initialize_wallet(token_trading_pairs=list(set(maker_assets))) + self._initialize_markets(market_names) + self.assets = set(maker_assets) + maker_data = [self.markets[exchange], trading_pair] + list(maker_assets) + self.market_trading_pair_tuples = [MarketTradingPairTuple(*maker_data)] + + strategy_logging_options = AvellanedaMarketMakingStrategy.OPTION_LOG_ALL + parameters_based_on_spread = c_map.get("parameters_based_on_spread").value + if parameters_based_on_spread: + risk_factor = order_book_depth_factor = order_amount_shape_factor = None + min_spread = c_map.get("min_spread").value / Decimal(100) + max_spread = c_map.get("max_spread").value / Decimal(100) + vol_to_spread_multiplier = c_map.get("vol_to_spread_multiplier").value + inventory_risk_aversion = c_map.get("inventory_risk_aversion").value + else: + min_spread = max_spread = vol_to_spread_multiplier = inventory_risk_aversion = None + order_book_depth_factor = c_map.get("order_book_depth_factor").value + risk_factor = c_map.get("risk_factor").value + order_amount_shape_factor = c_map.get("order_amount_shape_factor").value + closing_time = c_map.get("closing_time").value * Decimal(3600 * 24 * 1e3) + volatility_buffer_size = c_map.get("volatility_buffer_size").value + debug_csv_path = os.path.join(data_path(), + HummingbotApplication.main_application().strategy_file_name.rsplit('.', 1)[0] + + f"_{pd.Timestamp.now().strftime('%Y-%m-%d_%H-%M-%S')}.csv") + + self.strategy = AvellanedaMarketMakingStrategy( + market_info=MarketTradingPairTuple(*maker_data), + order_amount=order_amount, + order_optimization_enabled=order_optimization_enabled, + inventory_target_base_pct=inventory_target_base_pct, + order_refresh_time=order_refresh_time, + order_refresh_tolerance_pct=order_refresh_tolerance_pct, + filled_order_delay=filled_order_delay, + add_transaction_costs_to_orders=add_transaction_costs_to_orders, + logging_options=strategy_logging_options, + hb_app_notification=True, + parameters_based_on_spread=parameters_based_on_spread, + min_spread=min_spread, + max_spread=max_spread, + vol_to_spread_multiplier=vol_to_spread_multiplier, + inventory_risk_aversion=inventory_risk_aversion, + order_book_depth_factor=order_book_depth_factor, + risk_factor=risk_factor, + order_amount_shape_factor=order_amount_shape_factor, + closing_time=closing_time, + debug_csv_path=debug_csv_path, + volatility_buffer_size=volatility_buffer_size, + is_debug=False + ) + except Exception as e: + self._notify(str(e)) + self.logger().error("Unknown error during initialization.", exc_info=True) diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pxd b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pxd index af6556e069..ac99e62386 100755 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pxd +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pxd @@ -31,10 +31,12 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): dict _market_pairs int64_t _logging_options OrderIDMarketPairTracker _market_pair_tracker + bint _use_oracle_conversion_rate object _taker_to_maker_base_conversion_rate object _taker_to_maker_quote_conversion_rate bint _hb_app_notification list _maker_order_ids + double _last_conv_rates_logged cdef c_process_market_pair(self, object market_pair, diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx index 6bc8d1e66c..7f56c692ec 100755 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx @@ -9,6 +9,7 @@ from math import ( ceil ) from numpy import isnan +import pandas as pd from typing import ( List, Tuple, @@ -28,6 +29,8 @@ from hummingbot.strategy.strategy_base cimport StrategyBase from hummingbot.strategy.strategy_base import StrategyBase from .cross_exchange_market_pair import CrossExchangeMarketPair from .order_id_market_pair_tracker import OrderIDMarketPairTracker +from hummingbot.core.rate_oracle.rate_oracle import RateOracle +from hummingbot.client.performance import smart_round NaN = float("nan") s_decimal_zero = Decimal(0) @@ -73,6 +76,7 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): top_depth_tolerance: Decimal = Decimal(0), logging_options: int = OPTION_LOG_ALL, status_report_interval: float = 900, + use_oracle_conversion_rate: bool = False, taker_to_maker_base_conversion_rate: Decimal = Decimal("1"), taker_to_maker_quote_conversion_rate: Decimal = Decimal("1"), hb_app_notification: bool = False @@ -132,8 +136,10 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): self._status_report_interval = status_report_interval self._market_pair_tracker = OrderIDMarketPairTracker() self._adjust_orders_enabled = adjust_order_enabled + self._use_oracle_conversion_rate = use_oracle_conversion_rate self._taker_to_maker_base_conversion_rate = taker_to_maker_base_conversion_rate self._taker_to_maker_quote_conversion_rate = taker_to_maker_quote_conversion_rate + self._last_conv_rates_logged = 0 self._hb_app_notification = hb_app_notification self._maker_order_ids = [] @@ -167,6 +173,56 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): def logging_options(self, int64_t logging_options): self._logging_options = logging_options + def get_taker_to_maker_conversion_rate(self) -> Tuple[str, Decimal, str, Decimal]: + """ + Find conversion rates from taker market to maker market + :return: A tuple of quote pair symbol, quote conversion rate source, quote conversion rate, + base pair symbol, base conversion rate source, base conversion rate + """ + quote_rate = Decimal("1") + market_pairs = list(self._market_pairs.values())[0] + quote_pair = f"{market_pairs.taker.quote_asset}-{market_pairs.maker.quote_asset}" + quote_rate_source = "fixed" + if self._use_oracle_conversion_rate: + if market_pairs.taker.quote_asset != market_pairs.maker.quote_asset: + quote_rate_source = RateOracle.source.name + quote_rate = RateOracle.get_instance().rate(quote_pair) + else: + quote_rate = self._taker_to_maker_quote_conversion_rate + base_rate = Decimal("1") + base_pair = f"{market_pairs.taker.base_asset}-{market_pairs.maker.base_asset}" + base_rate_source = "fixed" + if self._use_oracle_conversion_rate: + if market_pairs.taker.base_asset != market_pairs.maker.base_asset: + base_rate_source = RateOracle.source.name + base_rate = RateOracle.get_instance().rate(base_pair) + else: + base_rate = self._taker_to_maker_base_conversion_rate + return quote_pair, quote_rate_source, quote_rate, base_pair, base_rate_source, base_rate + + def log_conversion_rates(self): + quote_pair, quote_rate_source, quote_rate, base_pair, base_rate_source, base_rate = \ + self.get_taker_to_maker_conversion_rate() + if quote_pair.split("-")[0] != quote_pair.split("-")[1]: + self.logger().info(f"{quote_pair} ({quote_rate_source}) conversion rate: {smart_round(quote_rate)}") + if base_pair.split("-")[0] != base_pair.split("-")[1]: + self.logger().info(f"{base_pair} ({base_rate_source}) conversion rate: {smart_round(base_rate)}") + + def oracle_status_df(self): + columns = ["Source", "Pair", "Rate"] + data = [] + quote_pair, quote_rate_source, quote_rate, base_pair, base_rate_source, base_rate = \ + self.get_taker_to_maker_conversion_rate() + if quote_pair.split("-")[0] != quote_pair.split("-")[1]: + data.extend([ + [quote_rate_source, quote_pair, smart_round(quote_rate)], + ]) + if base_pair.split("-")[0] != base_pair.split("-")[1]: + data.extend([ + [base_rate_source, base_pair, smart_round(base_rate)], + ]) + return pd.DataFrame(data=data, columns=columns) + def format_status(self) -> str: cdef: list lines = [] @@ -190,6 +246,11 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): lines.extend(["", " Markets:"] + [" " + line for line in str(markets_df).split("\n")]) + oracle_df = self.oracle_status_df() + if not oracle_df.empty: + lines.extend(["", " Rate conversion:"] + + [" " + line for line in str(oracle_df).split("\n")]) + assets_df = self.wallet_balance_data_frame([market_pair.maker, market_pair.taker]) lines.extend(["", " Assets:"] + [" " + line for line in str(assets_df).split("\n")]) @@ -305,6 +366,10 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): # Process each market pair independently. for market_pair in self._market_pairs.values(): self.c_process_market_pair(market_pair, market_pair_to_active_orders[market_pair]) + # log conversion rates every 5 minutes + if self._last_conv_rates_logged + (60. * 5) < self._current_timestamp: + self.log_conversion_rates() + self._last_conv_rates_logged = self._current_timestamp finally: self._last_timestamp = timestamp @@ -1070,13 +1135,16 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): object order_price = active_order.price ExchangeBase maker_market = market_pair.maker.market ExchangeBase taker_market = market_pair.taker.market - - object quote_asset_amount = maker_market.c_get_balance(market_pair.maker.quote_asset) if is_buy else \ - taker_market.c_get_balance(market_pair.taker.quote_asset) - object base_asset_amount = taker_market.c_get_balance(market_pair.taker.base_asset) if is_buy else \ - maker_market.c_get_balance(market_pair.maker.base_asset) object order_size_limit + quote_pair, quote_rate_source, quote_rate, base_pair, base_rate_source, base_rate = \ + self.get_taker_to_maker_conversion_rate() + + quote_asset_amount = maker_market.c_get_balance(market_pair.maker.quote_asset) if is_buy else \ + taker_market.c_get_balance(market_pair.taker.quote_asset) * quote_rate + base_asset_amount = taker_market.c_get_balance(market_pair.taker.base_asset) * base_rate if is_buy else \ + maker_market.c_get_balance(market_pair.maker.base_asset) + order_size_limit = min(base_asset_amount, quote_asset_amount / order_price) quantized_size_limit = maker_market.c_quantize_order_amount(active_order.trading_pair, order_size_limit) @@ -1096,7 +1164,15 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): """ Return price conversion rate for a taker market (to convert it into maker base asset value) """ - return self._taker_to_maker_quote_conversion_rate / self._taker_to_maker_base_conversion_rate + _, _, quote_rate, _, _, base_rate = self.get_taker_to_maker_conversion_rate() + return quote_rate / base_rate + # else: + # market_pairs = list(self._market_pairs.values())[0] + # quote_pair = f"{market_pairs.taker.quote_asset}-{market_pairs.maker.quote_asset}" + # base_pair = f"{market_pairs.taker.base_asset}-{market_pairs.maker.base_asset}" + # quote_rate = RateOracle.get_instance().rate(quote_pair) + # base_rate = RateOracle.get_instance().rate(base_pair) + # return quote_rate / base_rate cdef c_check_and_create_new_orders(self, object market_pair, bint has_active_bid, bint has_active_ask): """ @@ -1126,15 +1202,15 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): True, bid_size ) - effective_hedging_price_adjusted = effective_hedging_price * self.market_conversion_rate() + effective_hedging_price_adjusted = effective_hedging_price / self.market_conversion_rate() if self._logging_options & self.OPTION_LOG_CREATE_ORDER: self.log_with_clock( logging.INFO, f"({market_pair.maker.trading_pair}) Creating limit bid order for " f"{bid_size} {market_pair.maker.base_asset} at " f"{bid_price} {market_pair.maker.quote_asset}. " - f"Current hedging price: {effective_hedging_price} {market_pair.taker.quote_asset} " - f"(Rate adjusted: {effective_hedging_price_adjusted:.2f} {market_pair.taker.quote_asset})." + f"Current hedging price: {effective_hedging_price:.8f} {market_pair.maker.quote_asset} " + f"(Rate adjusted: {effective_hedging_price_adjusted:.8f} {market_pair.taker.quote_asset})." ) order_id = self.c_place_order(market_pair, True, True, bid_size, bid_price) else: @@ -1165,15 +1241,15 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): False, ask_size ) - effective_hedging_price_adjusted = effective_hedging_price + effective_hedging_price_adjusted = effective_hedging_price / self.market_conversion_rate() if self._logging_options & self.OPTION_LOG_CREATE_ORDER: self.log_with_clock( logging.INFO, f"({market_pair.maker.trading_pair}) Creating limit ask order for " f"{ask_size} {market_pair.maker.base_asset} at " f"{ask_price} {market_pair.maker.quote_asset}. " - f"Current hedging price: {effective_hedging_price} {market_pair.maker.quote_asset} " - f"(Rate adjusted: {effective_hedging_price_adjusted:.2f} {market_pair.maker.quote_asset})." + f"Current hedging price: {effective_hedging_price:.8f} {market_pair.maker.quote_asset} " + f"(Rate adjusted: {effective_hedging_price_adjusted:.8f} {market_pair.taker.quote_asset})." ) order_id = self.c_place_order(market_pair, False, True, ask_size, ask_price) else: diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py index 65eaaa0554..b621b0b554 100644 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py @@ -5,7 +5,8 @@ validate_decimal, validate_bool ) -from hummingbot.client.settings import required_exchanges, EXAMPLE_PAIRS +from hummingbot.client.config.config_helpers import parse_cvar_value +import hummingbot.client.settings as settings from decimal import Decimal from hummingbot.client.config.config_helpers import ( minimum_order_amount @@ -15,7 +16,7 @@ def maker_trading_pair_prompt(): maker_market = cross_exchange_market_making_config_map.get("maker_market").value - example = EXAMPLE_PAIRS.get(maker_market) + example = settings.EXAMPLE_PAIRS.get(maker_market) return "Enter the token trading pair you would like to trade on maker market: %s%s >>> " % ( maker_market, f" (e.g. {example})" if example else "", @@ -24,7 +25,7 @@ def maker_trading_pair_prompt(): def taker_trading_pair_prompt(): taker_market = cross_exchange_market_making_config_map.get("taker_market").value - example = EXAMPLE_PAIRS.get(taker_market) + example = settings.EXAMPLE_PAIRS.get(taker_market) return "Enter the token trading pair you would like to trade on taker market: %s%s >>> " % ( taker_market, f" (e.g. {example})" if example else "", @@ -68,7 +69,28 @@ async def validate_order_amount(value: str) -> Optional[str]: def taker_market_on_validated(value: str): - required_exchanges.append(value) + settings.required_exchanges.append(value) + + +def update_oracle_settings(value: str): + c_map = cross_exchange_market_making_config_map + if not (c_map["use_oracle_conversion_rate"].value is not None and + c_map["maker_market_trading_pair"].value is not None and + c_map["taker_market_trading_pair"].value is not None): + return + use_oracle = parse_cvar_value(c_map["use_oracle_conversion_rate"], c_map["use_oracle_conversion_rate"].value) + first_base, first_quote = c_map["maker_market_trading_pair"].value.split("-") + second_base, second_quote = c_map["taker_market_trading_pair"].value.split("-") + if use_oracle and (first_base != second_base or first_quote != second_quote): + settings.required_rate_oracle = True + settings.rate_oracle_pairs = [] + if first_base != second_base: + settings.rate_oracle_pairs.append(f"{second_base}-{first_base}") + if first_quote != second_quote: + settings.rate_oracle_pairs.append(f"{second_quote}-{first_quote}") + else: + settings.required_rate_oracle = False + settings.rate_oracle_pairs = [] cross_exchange_market_making_config_map = { @@ -81,7 +103,7 @@ def taker_market_on_validated(value: str): prompt="Enter your maker spot connector >>> ", prompt_on_new=True, validator=validate_exchange, - on_validated=lambda value: required_exchanges.append(value), + on_validated=lambda value: settings.required_exchanges.append(value), ), "taker_market": ConfigVar( key="taker_market", @@ -94,13 +116,15 @@ def taker_market_on_validated(value: str): key="maker_market_trading_pair", prompt=maker_trading_pair_prompt, prompt_on_new=True, - validator=validate_maker_market_trading_pair + validator=validate_maker_market_trading_pair, + on_validated=update_oracle_settings ), "taker_market_trading_pair": ConfigVar( key="taker_market_trading_pair", prompt=taker_trading_pair_prompt, prompt_on_new=True, - validator=validate_taker_market_trading_pair + validator=validate_taker_market_trading_pair, + on_validated=update_oracle_settings ), "min_profitability": ConfigVar( key="min_profitability", @@ -193,22 +217,29 @@ def taker_market_on_validated(value: str): required_if=lambda: False, validator=lambda v: validate_decimal(v, Decimal(0), Decimal(100), inclusive=False) ), + "use_oracle_conversion_rate": ConfigVar( + key="use_oracle_conversion_rate", + type_str="bool", + prompt="Do you want to use rate oracle on unmatched trading pairs? (Yes/No) >>> ", + prompt_on_new=True, + validator=lambda v: validate_bool(v), + on_validated=update_oracle_settings), "taker_to_maker_base_conversion_rate": ConfigVar( key="taker_to_maker_base_conversion_rate", prompt="Enter conversion rate for taker base asset value to maker base asset value, e.g. " - "if maker base asset is USD, taker is DAI and 1 USD is worth 1.25 DAI, " - "the conversion rate is 0.8 (1 / 1.25) >>> ", + "if maker base asset is USD and the taker is DAI, 1 DAI is valued at 1.25 USD, " + "the conversion rate is 1.25 >>> ", default=Decimal("1"), - validator=lambda v: validate_decimal(v, Decimal(0), Decimal("100"), inclusive=False), + validator=lambda v: validate_decimal(v, Decimal(0), inclusive=False), type_str="decimal" ), "taker_to_maker_quote_conversion_rate": ConfigVar( key="taker_to_maker_quote_conversion_rate", prompt="Enter conversion rate for taker quote asset value to maker quote asset value, e.g. " - "if taker quote asset is USD, maker is DAI and 1 USD is worth 1.25 DAI, " - "the conversion rate is 0.8 (1 / 1.25) >>> ", + "if maker quote asset is USD and the taker is DAI, 1 DAI is valued at 1.25 USD, " + "the conversion rate is 1.25 >>> ", default=Decimal("1"), - validator=lambda v: validate_decimal(v, Decimal(0), Decimal("100"), inclusive=False), + validator=lambda v: validate_decimal(v, Decimal(0), inclusive=False), type_str="decimal" ), } diff --git a/hummingbot/strategy/cross_exchange_market_making/start.py b/hummingbot/strategy/cross_exchange_market_making/start.py index c3ffe6ef98..a735d69863 100644 --- a/hummingbot/strategy/cross_exchange_market_making/start.py +++ b/hummingbot/strategy/cross_exchange_market_making/start.py @@ -28,6 +28,7 @@ def start(self): order_size_taker_balance_factor = xemm_map.get("order_size_taker_balance_factor").value / Decimal("100") order_size_portfolio_ratio_limit = xemm_map.get("order_size_portfolio_ratio_limit").value / Decimal("100") anti_hysteresis_duration = xemm_map.get("anti_hysteresis_duration").value + use_oracle_conversion_rate = xemm_map.get("use_oracle_conversion_rate").value taker_to_maker_base_conversion_rate = xemm_map.get("taker_to_maker_base_conversion_rate").value taker_to_maker_quote_conversion_rate = xemm_map.get("taker_to_maker_quote_conversion_rate").value @@ -83,6 +84,7 @@ def start(self): order_size_taker_balance_factor=order_size_taker_balance_factor, order_size_portfolio_ratio_limit=order_size_portfolio_ratio_limit, anti_hysteresis_duration=anti_hysteresis_duration, + use_oracle_conversion_rate=use_oracle_conversion_rate, taker_to_maker_base_conversion_rate=taker_to_maker_base_conversion_rate, taker_to_maker_quote_conversion_rate=taker_to_maker_quote_conversion_rate, hb_app_notification=True, diff --git a/hummingbot/strategy/perpetual_market_making/perpetual_market_making.pyx b/hummingbot/strategy/perpetual_market_making/perpetual_market_making.pyx index 7a842f6d2e..b5a635b37c 100644 --- a/hummingbot/strategy/perpetual_market_making/perpetual_market_making.pyx +++ b/hummingbot/strategy/perpetual_market_making/perpetual_market_making.pyx @@ -661,7 +661,7 @@ cdef class PerpetualMarketMakingStrategy(StrategyBase): for position in active_positions: if (ask_price > position.entry_price and position.amount > 0) or (bid_price < position.entry_price and position.amount < 0): # check if there is an active order to take profit, and create if none exists - profit_spread = self._long_profit_taking_spread if position.amount < 0 else self._short_profit_taking_spread + profit_spread = self._long_profit_taking_spread if position.amount > 0 else self._short_profit_taking_spread take_profit_price = position.entry_price * (Decimal("1") + profit_spread) if position.amount > 0 \ else position.entry_price * (Decimal("1") - profit_spread) price = market.c_quantize_order_price(self.trading_pair, take_profit_price) @@ -675,10 +675,10 @@ cdef class PerpetualMarketMakingStrategy(StrategyBase): size = market.c_quantize_order_amount(self.trading_pair, abs(position.amount)) if size > 0 and price > 0: if position.amount < 0: - self.logger().info(f"Creating profit taking buy order to lock profit on long position.") + self.logger().info(f"Creating profit taking buy order to lock profit on short position.") buys.append(PriceSize(price, size)) else: - self.logger().info(f"Creating profit taking sell order to lock profit on short position.") + self.logger().info(f"Creating profit taking sell order to lock profit on long position.") sells.append(PriceSize(price, size)) return Proposal(buys, sells) diff --git a/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py b/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py index 1501d16c6c..e488ec0f17 100644 --- a/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py +++ b/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py @@ -100,6 +100,11 @@ def validate_price_floor_ceiling(value: str) -> Optional[str]: return "Value must be more than 0 or -1 to disable this feature." +def on_validated_price_type(value: str): + if value == 'inventory_cost': + pure_market_making_config_map["inventory_price"].value = None + + def exchange_on_validated(value: str): required_exchanges.append(value) @@ -241,6 +246,7 @@ def exchange_on_validated(value: str): prompt="What is the price of your base asset inventory? ", type_str="decimal", validator=lambda v: validate_decimal(v, min_value=Decimal("0"), inclusive=True), + required_if=lambda: pure_market_making_config_map.get("price_type").value == "inventory_cost", default=Decimal("1"), ), "filled_order_delay": @@ -308,6 +314,7 @@ def exchange_on_validated(value: str): type_str="str", required_if=lambda: pure_market_making_config_map.get("price_source").value != "custom_api", default="mid_price", + on_validated=on_validated_price_type, validator=lambda s: None if s in {"mid_price", "last_price", "last_own_trade_price", diff --git a/hummingbot/templates/conf_arbitrage_strategy_TEMPLATE.yml b/hummingbot/templates/conf_arbitrage_strategy_TEMPLATE.yml index c383731e90..7459d5f5b6 100644 --- a/hummingbot/templates/conf_arbitrage_strategy_TEMPLATE.yml +++ b/hummingbot/templates/conf_arbitrage_strategy_TEMPLATE.yml @@ -2,7 +2,7 @@ ### Arbitrage strategy config ### ##################################### -template_version: 4 +template_version: 5 strategy: null # The following configuations are only required for the @@ -18,6 +18,10 @@ secondary_market_trading_pair: null # Expressed in percentage value, e.g. 1 = 1% target profit min_profitability: null +# Whether to use rate oracle on unmatched trading pairs +# Set this to either True or False +use_oracle_conversion_rate: null + # The conversion rate for secondary base asset value to primary base asset value. # e.g. if primary base asset is USD, secondary is DAI and 1 USD is worth 1.25 DAI, " # the conversion rate is 0.8 (1 / 1.25) diff --git a/hummingbot/templates/conf_avellaneda_market_making_strategy_TEMPLATE.yml b/hummingbot/templates/conf_avellaneda_market_making_strategy_TEMPLATE.yml new file mode 100644 index 0000000000..c4205052b4 --- /dev/null +++ b/hummingbot/templates/conf_avellaneda_market_making_strategy_TEMPLATE.yml @@ -0,0 +1,52 @@ +######################################################## +### Avellaneda market making strategy config ### +######################################################## + +template_version: 1 +strategy: null + +# Exchange and token parameters. +exchange: null + +# Token trading pair for the exchange, e.g. BTC-USDT +market: null + +# Time in seconds before cancelling and placing new orders. +# If the value is 60, the bot cancels active orders and placing new ones after a minute. +order_refresh_time: null + +# Whether to enable order optimization mode (true/false). +order_optimization_enabled: true + +# Time in seconds before replacing existing order with new orders at thesame price. +max_order_age: null + +# The spread (from mid price) to defer order refresh process to the next cycle. +# (Enter 1 to indicate 1%), value below 0, e.g. -1, is to disable this feature - not recommended. +order_refresh_tolerance_pct: null + +# Size of your bid and ask order. +order_amount: null + +# How long to wait before placing the next order in case your order gets filled. +filled_order_delay: null + +# Target base asset inventory percentage target to be maintained (for Inventory skew feature). +inventory_target_base_pct: null + +# Whether to enable adding transaction costs to order price calculation (true/false). +add_transaction_costs: null + +# Avellaneda - Stoikov algorithm parameters +parameters_based_on_spread: null +min_spread: null +max_spread: null +vol_to_spread_multiplier: null +inventory_risk_aversion: null +order_book_depth_factor: null +risk_factor: null +order_amount_shape_factor: null +closing_time: null + +# Buffer size used to store historic samples and calculate volatility +volatility_buffer_size: 60 \ No newline at end of file diff --git a/hummingbot/templates/conf_cross_exchange_market_making_strategy_TEMPLATE.yml b/hummingbot/templates/conf_cross_exchange_market_making_strategy_TEMPLATE.yml index dff79727cf..f435dbdbaa 100644 --- a/hummingbot/templates/conf_cross_exchange_market_making_strategy_TEMPLATE.yml +++ b/hummingbot/templates/conf_cross_exchange_market_making_strategy_TEMPLATE.yml @@ -2,7 +2,7 @@ ### Cross exchange market making strategy config ### ######################################################## -template_version: 4 +template_version: 5 strategy: null # The following configuations are only required for the @@ -60,6 +60,10 @@ order_size_taker_balance_factor: null # in terms of ratio of total portfolio value on both maker and taker markets order_size_portfolio_ratio_limit: null +# Whether to use rate oracle on unmatched trading pairs +# Set this to either True or False +use_oracle_conversion_rate: null + # The conversion rate for taker base asset value to maker base asset value. # e.g. if maker base asset is USD, taker is DAI and 1 USD is worth 1.25 DAI, " # the conversion rate is 0.8 (1 / 1.25) diff --git a/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml b/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml index ed08e2fa90..b8e79b7993 100644 --- a/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml +++ b/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml @@ -17,6 +17,9 @@ beaxy_taker_fee: coinbase_pro_maker_fee: coinbase_pro_taker_fee: +coinzoom_maker_fee: +coinzoom_taker_fee: + dydx_maker_fee: dydx_taker_fee: @@ -72,8 +75,11 @@ okex_taker_fee: balancer_maker_fee_amount: balancer_taker_fee_amount: -bitmax_maker_fee: -bitmax_taker_fee: +uniswap_maker_fee_amount: +uniswap_taker_fee_amount: + +ascend_ex_maker_fee: +ascend_ex_taker_fee: probit_maker_fee: probit_taker_fee: diff --git a/hummingbot/templates/conf_global_TEMPLATE.yml b/hummingbot/templates/conf_global_TEMPLATE.yml index 2d3af846f3..ad833aa312 100644 --- a/hummingbot/templates/conf_global_TEMPLATE.yml +++ b/hummingbot/templates/conf_global_TEMPLATE.yml @@ -3,7 +3,7 @@ ################################# # For more detailed information: https://docs.hummingbot.io -template_version: 19 +template_version: 20 # Exchange configs bamboo_relay_use_coordinator: false @@ -34,6 +34,10 @@ coinbase_pro_api_key: null coinbase_pro_secret_key: null coinbase_pro_passphrase: null +coinzoom_api_key: null +coinzoom_secret_key: null +coinzoom_username: null + dydx_eth_private_key: null dydx_node_address: null @@ -69,8 +73,8 @@ okex_api_key: null okex_secret_key: null okex_passphrase: null -bitmax_api_key: null -bitmax_secret_key: null +ascend_ex_api_key: null +ascend_ex_secret_key: null celo_address: null celo_password: null @@ -78,6 +82,9 @@ celo_password: null terra_wallet_address: null terra_wallet_seeds: null +digifinex_api_key: null +digifinex_secret_key: null + balancer_max_swaps: 4 probit_api_key: null @@ -91,13 +98,16 @@ ethereum_wallet: null ethereum_rpc_url: null ethereum_rpc_ws_url: null ethereum_chain_name: MAIN_NET -ethereum_token_list_url: null +ethereum_token_list_url: https://wispy-bird-88a7.uniswap.workers.dev/?url=http://tokens.1inch.eth.link # Kill switch kill_switch_enabled: null # The rate of performance at which you would want the bot to stop trading (-20 = 20%) kill_switch_rate: null +# What to auto-fill in the prompt after each import command (start/config) +autofill_import: null + # DEX active order cancellation 0x_active_cancels: false @@ -169,14 +179,7 @@ balance_asset_limit: # Fixed gas price (in Gwei) for Ethereum transactions manual_gas_price: -# To enable gas price lookup (true/false) -ethgasstation_gas_enabled: -# API key for defipulse.com gas station API -ethgasstation_api_key: -# Gas level you want to use for Ethereum transactions (fast, fastest, safeLow, average) -ethgasstation_gas_level: -# Refresh time for Ethereum gas price lookups (in seconds) -ethgasstation_refresh_time: + # Gateway API Configurations # default host to only use localhost # Port need to match the final installation port for Gateway @@ -190,8 +193,11 @@ heartbeat_interval_min: # a list of binance markets (for trades/pnl reporting) separated by ',' e.g. RLC-USDT,RLC-BTC binance_markets: +# A source for rate oracle, currently binance or coingecko rate_oracle_source: +# A universal token which to display tokens values in, e.g. USD,EUR,BTC global_token: +# A symbol for the global token, e.g. $, € global_token_symbol: \ No newline at end of file diff --git a/installation/docker-commands/create-gateway.sh b/installation/docker-commands/create-gateway.sh index c6db0bae8c..49fc98579c 100755 --- a/installation/docker-commands/create-gateway.sh +++ b/installation/docker-commands/create-gateway.sh @@ -40,7 +40,7 @@ else else echo "‼️ hummingbot_conf & hummingbot_certs directory missing from path $FOLDER" prompt_hummingbot_data_path - fi + fi if [[ -f "$FOLDER/hummingbot_certs/server_cert.pem" && -f "$FOLDER/hummingbot_certs/server_key.pem" && -f "$FOLDER/hummingbot_certs/ca_cert.pem" ]]; then echo @@ -79,70 +79,208 @@ do then HUMMINGBOT_INSTANCE_ID="$(echo -e "${value}" | tr -d '[:space:]')" fi - # chain - if [ "$key" == "ethereum_chain_name" ] + # +done < "$GLOBAL_CONFIG" +} +read_global_config + +# prompt to setup balancer, uniswap +prompt_ethereum_setup () { + read -p " Do you want to setup Balancer or Uniswap? [Y/N] (default \"Y\") >>> " PROCEED + if [[ "$PROCEED" == "Y" || "$PROCEED" == "y" || "$PROCEED" == "" ]] then - ETHEREUM_CHAIN="$(echo -e "${value}" | tr -d '[:space:]')" - # subgraph url - if [[ "$ETHEREUM_CHAIN" == "MAIN_NET" || "$ETHEREUM_CHAIN" == "main_net" || "$ETHEREUM_CHAIN" == "MAINNET" || "$ETHEREUM_CHAIN" == "mainnet" ]] + ETHEREUM_SETUP=true + echo + read -p " Enter Ethereum chain you want to use [mainnet/kovan] (default = \"mainnet\") >>> " ETHEREUM_CHAIN + # chain selection + if [ "$ETHEREUM_CHAIN" == "" ] + then + ETHEREUM_CHAIN="mainnet" + fi + if [[ "$ETHEREUM_CHAIN" != "mainnet" && "$ETHEREUM_CHAIN" != "kovan" ]] + then + echo "‼️ ERROR. Unsupported chains (mainnet/kovan). " + prompt_ethereum_setup + fi + # set subgraph url, exchange_proxy + if [[ "$ETHEREUM_CHAIN" == "mainnet" ]] then ETHEREUM_CHAIN="mainnet" REACT_APP_SUBGRAPH_URL="https://api.thegraph.com/subgraphs/name/balancer-labs/balancer" EXCHANGE_PROXY="0x3E66B66Fd1d0b02fDa6C811Da9E0547970DB2f21" else - ETHEREUM_CHAIN="kovan" - REACT_APP_SUBGRAPH_URL="https://api.thegraph.com/subgraphs/name/balancer-labs/balancer-kovan" - EXCHANGE_PROXY="0x4e67bf5bD28Dd4b570FBAFe11D0633eCbA2754Ec" + if [[ "$ETHEREUM_CHAIN" == "kovan" ]] + then + REACT_APP_SUBGRAPH_URL="https://api.thegraph.com/subgraphs/name/balancer-labs/balancer-kovan" + EXCHANGE_PROXY="0x4e67bf5bD28Dd4b570FBAFe11D0633eCbA2754Ec" + fi fi fi - # ethereum rpc url - if [ "$key" == "ethereum_rpc_url" ] +} +prompt_ethereum_setup + +# prompt to ethereum rpc +prompt_ethereum_rpc_setup () { + if [ "$ETHEREUM_RPC_URL" == "" ] then - ETHEREUM_RPC_URL="$(echo -e "${value}" | tr -d '[:space:]')" + read -p " Enter the Ethereum RPC node URL to connect to >>> " ETHEREUM_RPC_URL + if [ "$ETHEREUM_RPC_URL" == "" ] + then + prompt_ethereum_rpc_setup + fi + else + read -p " Use the this Ethereum RPC node ($ETHEREUM_RPC_URL) setup in Hummingbot client? [Y/N] (default = \"Y\") >>> " PROCEED + if [[ "$PROCEED" == "Y" || "$PROCEED" == "y" || "$PROCEED" == "" ]] + then + echo + else + ETHEREUM_RPC_URL="" + prompt_ethereum_rpc_setup + fi fi -done < "$GLOBAL_CONFIG" } -read_global_config +prompt_ethereum_rpc_setup -# prompt to setup balancer, uniswap -prompt_ethereum_setup () { - read -p " Do you want to setup Balancer/Uniswap/Perpetual Finance? [Y/N] >>> " PROCEED - if [[ "$PROCEED" == "Y" || "$PROCEED" == "y" ]] +# prompt to setup ethereum token list +prompt_token_list_source () { + echo + echo " Enter the token list url available at https://tokenlists.org/" + read -p " (default = \"https://wispy-bird-88a7.uniswap.workers.dev/?url=http://tokens.1inch.eth.link\") >>> " ETHEREUM_TOKEN_LIST_URL + if [ "$ETHEREUM_TOKEN_LIST_URL" == "" ] then + echo echo "ℹ️ Retrieving config from Hummingbot config file ... " ETHEREUM_SETUP=true + ETHEREUM_TOKEN_LIST_URL=https://wispy-bird-88a7.uniswap.workers.dev/?url=http://tokens.1inch.eth.link + fi +} +prompt_token_list_source + +# prompt to setup eth gas level +prompt_eth_gasstation_gas_level () { + echo + read -p " Enter gas level you want to use for Ethereum transactions (fast, fastest, safeLow, average) (default = \"fast\") >>> " ETH_GAS_STATION_GAS_LEVEL + if [ "$ETH_GAS_STATION_GAS_LEVEL" == "" ] + then + ETH_GAS_STATION_GAS_LEVEL=fast + else + if [[ "$ETH_GAS_STATION_GAS_LEVEL" != "fast" && "$ETH_GAS_STATION_GAS_LEVEL" != "fastest" && "$ETH_GAS_STATION_GAS_LEVEL" != "safeLow" && "$ETH_GAS_STATION_GAS_LEVEL" != "safelow" && "$ETH_GAS_STATION_GAS_LEVEL" != "average" ]] + then + prompt_eth_gasstation_gas_level + fi + fi +} + +# prompt to setup eth gas station +prompt_eth_gasstation_setup () { + echo + read -p " Enable dynamic Ethereum gas price lookup? [Y/N] (default = \"Y\") >>> " PROCEED + if [[ "$PROCEED" == "Y" || "$PROCEED" == "y" || "$PROCEED" == "" ]] + then + ENABLE_ETH_GAS_STATION=true + read -p " Enter API key for Eth Gas Station (https://ethgasstation.info/) >>> " ETH_GAS_STATION_API_KEY + if [ "$ETH_GAS_STATION_API_KEY" == "" ] + then + prompt_eth_gasstation_setup + else + # set gas level + prompt_eth_gasstation_gas_level + + # set refresh interval + read -p " Enter refresh time for Ethereum gas price lookup (in seconds) (default = \"120\") >>> " ETH_GAS_STATION_REFRESH_TIME + if [ "$ETH_GAS_STATION_REFRESH_TIME" == "" ] + then + ETH_GAS_STATION_REFRESH_TIME=120 + fi + fi + else + if [[ "$PROCEED" == "N" || "$PROCEED" == "n" ]] + then + ENABLE_ETH_GAS_STATION=false + # set manual gas price + read -p " Enter fixed gas price (in Gwei) you want to use for Ethereum transactions (default = \"100\") >>> " MANUAL_GAS_PRICE + if [ "$MANUAL_GAS_PRICE" == "" ] + then + MANUAL_GAS_PRICE=100 + fi + else + prompt_eth_gasstation_setup + fi + fi + echo +} +prompt_eth_gasstation_setup + +prompt_balancer_setup () { + # Ask the user for the Balancer specific settings + echo "ℹ️ Balancer setting " + read -p " Enter the maximum Balancer swap pool (default = \"4\") >>> " BALANCER_MAX_SWAPS + if [ "$BALANCER_MAX_SWAPS" == "" ] + then + BALANCER_MAX_SWAPS="4" echo fi } -prompt_ethereum_setup -# Ask the user for ethereum network -prompt_terra_network () { -read -p " Enter Terra chain you want to use [mainnet/testnet] (default = \"mainnet\") >>> " TERRA -# chain selection -if [ "$TERRA" == "" ] -then - TERRA="mainnet" -fi -if [[ "$TERRA" != "mainnet" && "$TERRA" != "testnet" ]] -then - echo "‼️ ERROR. Unsupported chains (mainnet/testnet). " - prompt_terra_network -fi -# setup chain params -if [[ "$TERRA" == "mainnet" ]] -then - TERRA_LCD_URL="https://lcd.terra.dev" - TERRA_CHAIN="columbus-4" -elif [ "$TERRA" == "testnet" ] +prompt_uniswap_setup () { + # Ask the user for the Uniswap specific settings + echo "ℹ️ Uniswap setting " + read -p " Enter the allowed slippage for swap transactions (default = \"1.5\") >>> " UNISWAP_SLIPPAGE + if [ "$UNISWAP_SLIPPAGE" == "" ] + then + UNISWAP_SLIPPAGE="1.5" + echo + fi +} + +if [[ "$ETHEREUM_SETUP" == true ]] then - TERRA_LCD_URL="https://tequila-lcd.terra.dev" - TERRA_CHAIN="tequila-0004" + prompt_balancer_setup + prompt_uniswap_setup fi + +prompt_xdai_setup () { + # Ask the user for the Uniswap specific settings + echo "ℹ️ XDAI setting " + read -p " Enter preferred XDAI rpc provider (default = \"https://rpc.xdaichain.com\") >>> " XDAI_PROVIDER + if [ "$XDAI_PROVIDER" == "" ] + then + XDAI_PROVIDER="https://rpc.xdaichain.com" + echo + fi } +prompt_xdai_setup + +# Ask the user for ethereum network +prompt_terra_network () { + echo + read -p " Enter Terra chain you want to use [mainnet/testnet] (default = \"mainnet\") >>> " TERRA + # chain selection + if [ "$TERRA" == "" ] + then + TERRA="mainnet" + fi + if [[ "$TERRA" != "mainnet" && "$TERRA" != "testnet" ]] + then + echo "‼️ ERROR. Unsupported chains (mainnet/testnet). " + prompt_terra_network + fi + # setup chain params + if [[ "$TERRA" == "mainnet" ]] + then + TERRA_LCD_URL="https://lcd.terra.dev" + TERRA_CHAIN="columbus-4" + elif [ "$TERRA" == "testnet" ] + then + TERRA_LCD_URL="https://tequila-lcd.terra.dev" + TERRA_CHAIN="tequila-0004" + fi +} + prompt_terra_setup () { - read -p " Do you want to setup Terra? [Y/N] >>> " PROCEED - if [[ "$PROCEED" == "Y" || "$PROCEED" == "y" ]] + echo + read -p " Do you want to setup Terra? [Y/N] (default \"Y\") >>> " PROCEED + if [[ "$PROCEED" == "Y" || "$PROCEED" == "y" || "$PROCEED" == "" ]] then TERRA_SETUP=true prompt_terra_network @@ -202,9 +340,17 @@ echo printf "%30s %5s\n" "Hummingbot Instance ID:" "$HUMMINGBOT_INSTANCE_ID" printf "%30s %5s\n" "Ethereum Chain:" "$ETHEREUM_CHAIN" printf "%30s %5s\n" "Ethereum RPC URL:" "$ETHEREUM_RPC_URL" +printf "%30s %5s\n" "Ethereum Token List URL:" "$ETHEREUM_TOKEN_LIST_URL" +printf "%30s %5s\n" "Manual Gas Price:" "$MANUAL_GAS_PRICE" +printf "%30s %5s\n" "Enable Eth Gas Station:" "$ENABLE_ETH_GAS_STATION" +printf "%30s %5s\n" "Eth Gas Station API:" "$ETH_GAS_STATION_API_KEY" +printf "%30s %5s\n" "Eth Gas Station Level:" "$ETH_GAS_STATION_GAS_LEVEL" +printf "%30s %5s\n" "Eth Gas Station Refresh Interval:" "$ETH_GAS_STATION_REFRESH_TIME" printf "%30s %5s\n" "Balancer Subgraph:" "$REACT_APP_SUBGRAPH_URL" printf "%30s %5s\n" "Balancer Exchange Proxy:" "$EXCHANGE_PROXY" +printf "%30s %5s\n" "Balancer Max Swaps:" "$BALANCER_MAX_SWAPS" printf "%30s %5s\n" "Uniswap Router:" "$UNISWAP_ROUTER" +printf "%30s %5s\n" "Uniswap Allowed Slippage:" "$UNISWAP_SLIPPAGE" printf "%30s %5s\n" "Terra Chain:" "$TERRA" printf "%30s %5s\n" "Gateway Log Path:" "$LOG_PATH" printf "%30s %5s\n" "Gateway Cert Path:" "$CERT_PATH" @@ -220,13 +366,46 @@ echo "NODE_ENV=prod" >> $ENV_FILE echo "PORT=$PORT" >> $ENV_FILE echo "" >> $ENV_FILE echo "HUMMINGBOT_INSTANCE_ID=$HUMMINGBOT_INSTANCE_ID" >> $ENV_FILE + +# ethereum config +echo "" >> $ENV_FILE +echo "# Ethereum Settings" >> $ENV_FILE echo "ETHEREUM_CHAIN=$ETHEREUM_CHAIN" >> $ENV_FILE echo "ETHEREUM_RPC_URL=$ETHEREUM_RPC_URL" >> $ENV_FILE +echo "ETHEREUM_TOKEN_LIST_URL=$ETHEREUM_TOKEN_LIST_URL" >> $ENV_FILE +echo "" >> $ENV_FILE +echo "ENABLE_ETH_GAS_STATION=$ENABLE_ETH_GAS_STATION" >> $ENV_FILE +echo "ETH_GAS_STATION_API_KEY=$ETH_GAS_STATION_API_KEY" >> $ENV_FILE +echo "ETH_GAS_STATION_GAS_LEVEL=$ETH_GAS_STATION_GAS_LEVEL" >> $ENV_FILE +echo "ETH_GAS_STATION_REFRESH_TIME=$ETH_GAS_STATION_REFRESH_TIME" >> $ENV_FILE +echo "MANUAL_GAS_PRICE=$MANUAL_GAS_PRICE" >> $ENV_FILE + +# balancer config +echo "" >> $ENV_FILE +echo "# Balancer Settings" >> $ENV_FILE echo "REACT_APP_SUBGRAPH_URL=$REACT_APP_SUBGRAPH_URL" >> $ENV_FILE # must used "REACT_APP_SUBGRAPH_URL" for balancer-sor echo "EXCHANGE_PROXY=$EXCHANGE_PROXY" >> $ENV_FILE +echo "BALANCER_MAX_SWAPS=$BALANCER_MAX_SWAPS" >> $ENV_FILE + +# uniswap config +echo "" >> $ENV_FILE +echo "# Uniswap Settings" >> $ENV_FILE echo "UNISWAP_ROUTER=$UNISWAP_ROUTER" >> $ENV_FILE +echo "UNISWAP_ALLOWED_SLIPPAGE=$UNISWAP_SLIPPAGE" >> $ENV_FILE +echo "UNISWAP_NO_RESERVE_CHECK_INTERVAL=300000" >> $ENV_FILE +echo "UNISWAP_PAIRS_CACHE_TIME=1000" >> $ENV_FILE + +# terra config +echo "" >> $ENV_FILE +echo "# Terra Settings" >> $ENV_FILE echo "TERRA_LCD_URL=$TERRA_LCD_URL" >> $ENV_FILE echo "TERRA_CHAIN=$TERRA_CHAIN" >> $ENV_FILE + +# perpeptual finance config +echo "" >> $ENV_FILE +echo "# Perpeptual Settings" >> $ENV_FILE +echo "XDAI_PROVIDER=$XDAI_PROVIDER" >> $ENV_FILE + echo "" >> $ENV_FILE prompt_proceed () { @@ -234,7 +413,12 @@ prompt_proceed () { read -p " Do you want to proceed with installation? [Y/N] >>> " PROCEED if [ "$PROCEED" == "" ] then - PROCEED="Y" + prompt_proceed + else + if [[ "$PROCEED" != "Y" && "$PROCEED" != "y" ]] + then + PROCEED="N" + fi fi } diff --git a/setup.py b/setup.py index 083859ad4c..08ece91061 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from setuptools import setup +from setuptools.command.build_ext import build_ext from Cython.Build import cythonize import numpy as np import os @@ -17,6 +18,16 @@ os.environ["CFLAGS"] = "-std=c++11" +# Avoid a gcc warning below: +# cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid +# for C/ObjC but not for C++ +class BuildExt(build_ext): + def build_extensions(self): + if os.name != "nt" and '-Wstrict-prototypes' in self.compiler.compiler_so: + self.compiler.compiler_so.remove('-Wstrict-prototypes') + super().build_extensions() + + def main(): cpu_count = os.cpu_count() or 8 version = "20210309" @@ -39,11 +50,13 @@ def main(): "hummingbot.connector.connector.balancer", "hummingbot.connector.connector.terra", "hummingbot.connector.exchange", + "hummingbot.connector.exchange.ascend_ex", "hummingbot.connector.exchange.binance", "hummingbot.connector.exchange.bitfinex", "hummingbot.connector.exchange.bittrex", "hummingbot.connector.exchange.bamboo_relay", "hummingbot.connector.exchange.coinbase_pro", + "hummingbot.connector.exchange.coinzoom", "hummingbot.connector.exchange.dydx", "hummingbot.connector.exchange.huobi", "hummingbot.connector.exchange.radar_relay", @@ -56,7 +69,6 @@ def main(): "hummingbot.connector.exchange.dolomite", "hummingbot.connector.exchange.eterbase", "hummingbot.connector.exchange.beaxy", - "hummingbot.connector.exchange.bitmax", "hummingbot.connector.exchange.hitbtc", "hummingbot.connector.derivative", "hummingbot.connector.derivative.binance_perpetual", @@ -165,6 +177,7 @@ def main(): "bin/hummingbot.py", "bin/hummingbot_quickstart.py" ], + cmdclass={'build_ext': BuildExt}, ) diff --git a/test/connector/connector/balancer/test_balancer_connector.py b/test/connector/connector/balancer/test_balancer_connector.py index 151d0e403e..66ad53df97 100644 --- a/test/connector/connector/balancer/test_balancer_connector.py +++ b/test/connector/connector/balancer/test_balancer_connector.py @@ -28,8 +28,7 @@ global_config_map['gateway_api_host'].value = "localhost" global_config_map['gateway_api_port'].value = 5000 -global_config_map['ethgasstation_gas_enabled'].value = False -global_config_map['manual_gas_price'].value = 50 +global_config_map['ethereum_token_list_url'].value = "https://defi.cmc.eth.link" global_config_map.get("ethereum_chain_name").value = "kovan" trading_pair = "WETH-DAI" @@ -96,6 +95,14 @@ def setUp(self): for event_tag in self.events: self.connector.add_listener(event_tag, self.event_logger) + def test_fetch_trading_pairs(self): + asyncio.get_event_loop().run_until_complete(self._test_fetch_trading_pairs()) + + async def _test_fetch_trading_pairs(self): + pairs = await BalancerConnector.fetch_trading_pairs() + print(pairs) + self.assertGreater(len(pairs), 0) + def test_update_balances(self): all_bals = self.connector.get_all_balances() for token, bal in all_bals.items(): diff --git a/test/connector/exchange/bitmax/.gitignore b/test/connector/exchange/ascend_ex/.gitignore similarity index 100% rename from test/connector/exchange/bitmax/.gitignore rename to test/connector/exchange/ascend_ex/.gitignore diff --git a/test/connector/exchange/ascend_ex/__init__.py b/test/connector/exchange/ascend_ex/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/connector/exchange/bitmax/test_bitmax_auth.py b/test/connector/exchange/ascend_ex/test_ascend_ex_auth.py similarity index 72% rename from test/connector/exchange/bitmax/test_bitmax_auth.py rename to test/connector/exchange/ascend_ex/test_ascend_ex_auth.py index 46a68bfae7..59d866ad87 100644 --- a/test/connector/exchange/bitmax/test_bitmax_auth.py +++ b/test/connector/exchange/ascend_ex/test_ascend_ex_auth.py @@ -9,21 +9,22 @@ import logging from os.path import join, realpath from typing import Dict, Any -from hummingbot.connector.exchange.bitmax.bitmax_auth import BitmaxAuth +from hummingbot.connector.exchange.ascend_ex.ascend_ex_auth import AscendExAuth from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL -from hummingbot.connector.exchange.bitmax.bitmax_constants import REST_URL, getWsUrlPriv +from hummingbot.connector.exchange.ascend_ex.ascend_ex_constants import REST_URL +from hummingbot.connector.exchange.ascend_ex.ascend_ex_util import get_rest_url_private sys.path.insert(0, realpath(join(__file__, "../../../../../"))) logging.basicConfig(level=METRICS_LOG_LEVEL) -class TestAuth(unittest.TestCase): +class TestAscendExAuth(unittest.TestCase): @classmethod def setUpClass(cls): cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() - api_key = conf.bitmax_api_key - secret_key = conf.bitmax_secret_key - cls.auth = BitmaxAuth(api_key, secret_key) + api_key = conf.ascend_ex_api_key + secret_key = conf.ascend_ex_secret_key + cls.auth = AscendExAuth(api_key, secret_key) async def rest_auth(self) -> Dict[Any, Any]: headers = { @@ -37,7 +38,7 @@ async def ws_auth(self) -> Dict[Any, Any]: info = await self.rest_auth() accountGroup = info.get("data").get("accountGroup") headers = self.auth.get_auth_headers("stream") - ws = await websockets.connect(f"{getWsUrlPriv(accountGroup)}/stream", extra_headers=headers) + ws = await websockets.connect(f"{get_rest_url_private(accountGroup)}/stream", extra_headers=headers) raw_msg = await asyncio.wait_for(ws.recv(), 5000) msg = ujson.loads(raw_msg) diff --git a/test/connector/exchange/bitmax/test_bitmax_exchange.py b/test/connector/exchange/ascend_ex/test_ascend_ex_exchange.py similarity index 97% rename from test/connector/exchange/bitmax/test_bitmax_exchange.py rename to test/connector/exchange/ascend_ex/test_ascend_ex_exchange.py index ca642fe385..e3c928520d 100644 --- a/test/connector/exchange/bitmax/test_bitmax_exchange.py +++ b/test/connector/exchange/ascend_ex/test_ascend_ex_exchange.py @@ -1,15 +1,17 @@ from os.path import join, realpath import sys; sys.path.insert(0, realpath(join(__file__, "../../../../../"))) + import asyncio -import logging -from decimal import Decimal -import unittest +import conf import contextlib -import time +import logging +import math import os +import time +import unittest + +from decimal import Decimal from typing import List -import conf -import math from hummingbot.core.clock import Clock, ClockMode from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL @@ -33,15 +35,15 @@ from hummingbot.model.order import Order from hummingbot.model.trade_fill import TradeFill from hummingbot.connector.markets_recorder import MarketsRecorder -from hummingbot.connector.exchange.bitmax.bitmax_exchange import BitmaxExchange +from hummingbot.connector.exchange.ascend_ex.ascend_ex_exchange import AscendExExchange logging.basicConfig(level=METRICS_LOG_LEVEL) -API_KEY = conf.bitmax_api_key -API_SECRET = conf.bitmax_secret_key +API_KEY = conf.ascend_ex_api_key +API_SECRET = conf.ascend_ex_secret_key -class BitmaxExchangeUnitTest(unittest.TestCase): +class AscendExExchangeUnitTest(unittest.TestCase): events: List[MarketEvent] = [ MarketEvent.BuyOrderCompleted, MarketEvent.SellOrderCompleted, @@ -52,7 +54,7 @@ class BitmaxExchangeUnitTest(unittest.TestCase): MarketEvent.OrderCancelled, MarketEvent.OrderFailure ] - connector: BitmaxExchange + connector: AscendExExchange event_logger: EventLogger trading_pair = "BTC-USDT" base_token, quote_token = trading_pair.split("-") @@ -65,13 +67,13 @@ def setUpClass(cls): cls.ev_loop = asyncio.get_event_loop() cls.clock: Clock = Clock(ClockMode.REALTIME) - cls.connector: BitmaxExchange = BitmaxExchange( - bitmax_api_key=API_KEY, - bitmax_secret_key=API_SECRET, + cls.connector: AscendExExchange = AscendExExchange( + ascend_ex_api_key=API_KEY, + ascend_ex_secret_key=API_SECRET, trading_pairs=[cls.trading_pair], trading_required=True ) - print("Initializing Bitmax market... this will take about a minute.") + print("Initializing AscendEx exchange... this will take about a minute.") cls.clock.add_iterator(cls.connector) cls.stack: contextlib.ExitStack = contextlib.ExitStack() cls._clock = cls.stack.enter_context(cls.clock) @@ -336,7 +338,7 @@ def test_orders_saving_and_restoration(self): self.clock.remove_iterator(self.connector) for event_tag in self.events: self.connector.remove_listener(event_tag, self.event_logger) - new_connector = BitmaxExchange(API_KEY, API_SECRET, [self.trading_pair], True) + new_connector = AscendExExchange(API_KEY, API_SECRET, [self.trading_pair], True) for event_tag in self.events: new_connector.add_listener(event_tag, self.event_logger) recorder.stop() diff --git a/test/connector/exchange/bitmax/test_bitmax_order_book_tracker.py b/test/connector/exchange/ascend_ex/test_ascend_ex_order_book_tracker.py similarity index 88% rename from test/connector/exchange/bitmax/test_bitmax_order_book_tracker.py rename to test/connector/exchange/ascend_ex/test_ascend_ex_order_book_tracker.py index 331860fbba..ed19775c0d 100755 --- a/test/connector/exchange/bitmax/test_bitmax_order_book_tracker.py +++ b/test/connector/exchange/ascend_ex/test_ascend_ex_order_book_tracker.py @@ -9,8 +9,8 @@ from typing import Dict, Optional, List from hummingbot.core.event.event_logger import EventLogger from hummingbot.core.event.events import OrderBookEvent, OrderBookTradeEvent, TradeType -from hummingbot.connector.exchange.bitmax.bitmax_order_book_tracker import BitmaxOrderBookTracker -from hummingbot.connector.exchange.bitmax.bitmax_api_order_book_data_source import BitmaxAPIOrderBookDataSource +from hummingbot.connector.exchange.ascend_ex.ascend_ex_order_book_tracker import AscendExOrderBookTracker +from hummingbot.connector.exchange.ascend_ex.ascend_ex_api_order_book_data_source import AscendExAPIOrderBookDataSource from hummingbot.core.data_type.order_book import OrderBook from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL @@ -19,8 +19,8 @@ logging.basicConfig(level=METRICS_LOG_LEVEL) -class BitmaxOrderBookTrackerUnitTest(unittest.TestCase): - order_book_tracker: Optional[BitmaxOrderBookTracker] = None +class AscendExOrderBookTrackerUnitTest(unittest.TestCase): + order_book_tracker: Optional[AscendExOrderBookTracker] = None events: List[OrderBookEvent] = [ OrderBookEvent.TradeEvent ] @@ -32,7 +32,7 @@ class BitmaxOrderBookTrackerUnitTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() - cls.order_book_tracker: BitmaxOrderBookTracker = BitmaxOrderBookTracker(cls.trading_pairs) + cls.order_book_tracker: AscendExOrderBookTracker = AscendExOrderBookTracker(cls.trading_pairs) cls.order_book_tracker.start() cls.ev_loop.run_until_complete(cls.wait_til_tracker_ready()) @@ -96,7 +96,7 @@ def test_tracker_integrity(self): def test_api_get_last_traded_prices(self): prices = self.ev_loop.run_until_complete( - BitmaxAPIOrderBookDataSource.get_last_traded_prices(["BTC-USDT", "LTC-BTC"])) + AscendExAPIOrderBookDataSource.get_last_traded_prices(["BTC-USDT", "LTC-BTC"])) for key, value in prices.items(): print(f"{key} last_trade_price: {value}") self.assertGreater(prices["BTC-USDT"], 1000) diff --git a/test/connector/exchange/bitmax/test_bitmax_user_stream_tracker.py b/test/connector/exchange/ascend_ex/test_ascend_ex_user_stream_tracker.py similarity index 58% rename from test/connector/exchange/bitmax/test_bitmax_user_stream_tracker.py rename to test/connector/exchange/ascend_ex/test_ascend_ex_user_stream_tracker.py index 1b4cb5c84b..ce107d09f2 100644 --- a/test/connector/exchange/bitmax/test_bitmax_user_stream_tracker.py +++ b/test/connector/exchange/ascend_ex/test_ascend_ex_user_stream_tracker.py @@ -6,8 +6,8 @@ import conf from os.path import join, realpath -from hummingbot.connector.exchange.bitmax.bitmax_user_stream_tracker import BitmaxUserStreamTracker -from hummingbot.connector.exchange.bitmax.bitmax_auth import BitmaxAuth +from hummingbot.connector.exchange.ascend_ex.ascend_ex_user_stream_tracker import AscendExUserStreamTracker +from hummingbot.connector.exchange.ascend_ex.ascend_ex_auth import AscendExAuth from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL @@ -16,17 +16,17 @@ logging.basicConfig(level=METRICS_LOG_LEVEL) -class BitmaxUserStreamTrackerUnitTest(unittest.TestCase): - api_key = conf.bitmax_api_key - api_secret = conf.bitmax_secret_key +class AscendExUserStreamTrackerUnitTest(unittest.TestCase): + api_key = conf.ascend_ex_api_key + api_secret = conf.ascend_ex_secret_key @classmethod def setUpClass(cls): cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() - cls.bitmax_auth = BitmaxAuth(cls.api_key, cls.api_secret) + cls.ascend_ex_auth = AscendExAuth(cls.api_key, cls.api_secret) cls.trading_pairs = ["BTC-USDT"] - cls.user_stream_tracker: BitmaxUserStreamTracker = BitmaxUserStreamTracker( - bitmax_auth=cls.bitmax_auth, trading_pairs=cls.trading_pairs) + cls.user_stream_tracker: AscendExUserStreamTracker = AscendExUserStreamTracker( + ascend_ex_auth=cls.ascend_ex_auth, trading_pairs=cls.trading_pairs) cls.user_stream_tracker_task: asyncio.Task = safe_ensure_future(cls.user_stream_tracker.start()) def test_user_stream(self): diff --git a/test/connector/exchange/coinzoom/.gitignore b/test/connector/exchange/coinzoom/.gitignore new file mode 100644 index 0000000000..23d9952b8c --- /dev/null +++ b/test/connector/exchange/coinzoom/.gitignore @@ -0,0 +1 @@ +backups \ No newline at end of file diff --git a/test/connector/exchange/coinzoom/__init__.py b/test/connector/exchange/coinzoom/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/connector/exchange/coinzoom/test_coinzoom_auth.py b/test/connector/exchange/coinzoom/test_coinzoom_auth.py new file mode 100644 index 0000000000..f2573e4460 --- /dev/null +++ b/test/connector/exchange/coinzoom/test_coinzoom_auth.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +import sys +import asyncio +import unittest +import aiohttp +import conf +import logging +from async_timeout import timeout +from os.path import join, realpath +from typing import Dict, Any +from hummingbot.connector.exchange.coinzoom.coinzoom_auth import CoinzoomAuth +from hummingbot.connector.exchange.coinzoom.coinzoom_websocket import CoinzoomWebsocket +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL +from hummingbot.connector.exchange.coinzoom.coinzoom_constants import Constants + +sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +logging.basicConfig(level=METRICS_LOG_LEVEL) + + +class TestAuth(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + api_key = conf.coinzoom_api_key + secret_key = conf.coinzoom_secret_key + api_username = conf.coinzoom_username + cls.auth = CoinzoomAuth(api_key, secret_key, api_username) + + async def rest_auth(self) -> Dict[Any, Any]: + endpoint = Constants.ENDPOINT['USER_BALANCES'] + headers = self.auth.get_headers() + response = await aiohttp.ClientSession().get(f"{Constants.REST_URL}/{endpoint}", headers=headers) + return await response.json() + + async def ws_auth(self) -> Dict[Any, Any]: + ws = CoinzoomWebsocket(self.auth) + await ws.connect() + user_ws_streams = {Constants.WS_SUB["USER_ORDERS_TRADES"]: {}} + async with timeout(30): + await ws.subscribe(user_ws_streams) + async for response in ws.on_message(): + if ws.is_subscribed: + return True + return False + + def test_rest_auth(self): + result = self.ev_loop.run_until_complete(self.rest_auth()) + if len(result) == 0 or "currency" not in result[0].keys(): + print(f"Unexpected response for API call: {result}") + assert "currency" in result[0].keys() + + def test_ws_auth(self): + subscribed = self.ev_loop.run_until_complete(self.ws_auth()) + assert subscribed is True diff --git a/test/connector/exchange/coinzoom/test_coinzoom_exchange.py b/test/connector/exchange/coinzoom/test_coinzoom_exchange.py new file mode 100644 index 0000000000..979b4651b6 --- /dev/null +++ b/test/connector/exchange/coinzoom/test_coinzoom_exchange.py @@ -0,0 +1,442 @@ +from os.path import join, realpath +import sys; sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +import asyncio +import logging +from decimal import Decimal +import unittest +import contextlib +import time +import os +from typing import List +import conf +import math +from async_timeout import timeout + +from hummingbot.core.clock import Clock, ClockMode +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL +from hummingbot.core.utils.async_utils import safe_gather, safe_ensure_future +from hummingbot.core.event.event_logger import EventLogger +from hummingbot.core.event.events import ( + BuyOrderCompletedEvent, + BuyOrderCreatedEvent, + MarketEvent, + OrderFilledEvent, + OrderType, + SellOrderCompletedEvent, + SellOrderCreatedEvent, + OrderCancelledEvent +) +from hummingbot.model.sql_connection_manager import ( + SQLConnectionManager, + SQLConnectionType +) +from hummingbot.model.market_state import MarketState +from hummingbot.model.order import Order +from hummingbot.model.trade_fill import TradeFill +from hummingbot.connector.markets_recorder import MarketsRecorder +from hummingbot.connector.exchange.coinzoom.coinzoom_exchange import CoinzoomExchange + +logging.basicConfig(level=METRICS_LOG_LEVEL) + +API_KEY = conf.coinzoom_api_key +API_SECRET = conf.coinzoom_secret_key +API_USERNAME = conf.coinzoom_username + + +class CoinzoomExchangeUnitTest(unittest.TestCase): + events: List[MarketEvent] = [ + MarketEvent.BuyOrderCompleted, + MarketEvent.SellOrderCompleted, + MarketEvent.OrderFilled, + MarketEvent.TransactionFailure, + MarketEvent.BuyOrderCreated, + MarketEvent.SellOrderCreated, + MarketEvent.OrderCancelled, + MarketEvent.OrderFailure + ] + connector: CoinzoomExchange + event_logger: EventLogger + trading_pair = "BTC-USD" + base_token, quote_token = trading_pair.split("-") + stack: contextlib.ExitStack + + @classmethod + def setUpClass(cls): + global MAINNET_RPC_URL + + cls.ev_loop = asyncio.get_event_loop() + + cls.clock: Clock = Clock(ClockMode.REALTIME) + cls.connector: CoinzoomExchange = CoinzoomExchange( + coinzoom_api_key=API_KEY, + coinzoom_secret_key=API_SECRET, + coinzoom_username=API_USERNAME, + trading_pairs=[cls.trading_pair], + trading_required=True + ) + print("Initializing Coinzoom market... this will take about a minute.") + cls.clock.add_iterator(cls.connector) + cls.stack: contextlib.ExitStack = contextlib.ExitStack() + cls._clock = cls.stack.enter_context(cls.clock) + cls.ev_loop.run_until_complete(cls.wait_til_ready()) + print("Ready.") + + @classmethod + def tearDownClass(cls) -> None: + cls.stack.close() + + @classmethod + async def wait_til_ready(cls, connector = None): + if connector is None: + connector = cls.connector + async with timeout(90): + while True: + now = time.time() + next_iteration = now // 1.0 + 1 + if connector.ready: + break + else: + await cls._clock.run_til(next_iteration) + await asyncio.sleep(1.0) + + def setUp(self): + self.db_path: str = realpath(join(__file__, "../connector_test.sqlite")) + try: + os.unlink(self.db_path) + except FileNotFoundError: + pass + + self.event_logger = EventLogger() + for event_tag in self.events: + self.connector.add_listener(event_tag, self.event_logger) + + def tearDown(self): + for event_tag in self.events: + self.connector.remove_listener(event_tag, self.event_logger) + self.event_logger = None + + async def run_parallel_async(self, *tasks): + future: asyncio.Future = safe_ensure_future(safe_gather(*tasks)) + while not future.done(): + now = time.time() + next_iteration = now // 1.0 + 1 + await self._clock.run_til(next_iteration) + await asyncio.sleep(1.0) + return future.result() + + def run_parallel(self, *tasks): + return self.ev_loop.run_until_complete(self.run_parallel_async(*tasks)) + + def _place_order(self, is_buy, amount, order_type, price, ex_order_id) -> str: + if is_buy: + cl_order_id = self.connector.buy(self.trading_pair, amount, order_type, price) + else: + cl_order_id = self.connector.sell(self.trading_pair, amount, order_type, price) + return cl_order_id + + def _cancel_order(self, cl_order_id, connector=None): + if connector is None: + connector = self.connector + return connector.cancel(self.trading_pair, cl_order_id) + + def test_estimate_fee(self): + maker_fee = self.connector.estimate_fee_pct(True) + self.assertAlmostEqual(maker_fee, Decimal("0.002")) + taker_fee = self.connector.estimate_fee_pct(False) + self.assertAlmostEqual(taker_fee, Decimal("0.0026")) + + def test_buy_and_sell(self): + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.02") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + quote_bal = self.connector.get_available_balance(self.quote_token) + base_bal = self.connector.get_available_balance(self.base_token) + + order_id = self._place_order(True, amount, OrderType.LIMIT, price, 1) + order_completed_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(5)) + trade_events = [t for t in self.event_logger.event_log if isinstance(t, OrderFilledEvent)] + base_amount_traded = sum(t.amount for t in trade_events) + quote_amount_traded = sum(t.amount * t.price for t in trade_events) + + self.assertTrue([evt.order_type == OrderType.LIMIT for evt in trade_events]) + self.assertEqual(order_id, order_completed_event.order_id) + self.assertEqual(amount, order_completed_event.base_asset_amount) + self.assertEqual("BTC", order_completed_event.base_asset) + self.assertEqual("USD", order_completed_event.quote_asset) + self.assertAlmostEqual(base_amount_traded, order_completed_event.base_asset_amount) + self.assertAlmostEqual(quote_amount_traded, order_completed_event.quote_asset_amount) + self.assertGreater(order_completed_event.fee_amount, Decimal(0)) + self.assertTrue(any([isinstance(event, BuyOrderCreatedEvent) and str(event.order_id) == str(order_id) + for event in self.event_logger.event_log])) + + # check available quote balance gets updated, we need to wait a bit for the balance message to arrive + expected_quote_bal = quote_bal - quote_amount_traded + # self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.ev_loop.run_until_complete(self.connector._update_balances()) + self.assertAlmostEqual(expected_quote_bal, self.connector.get_available_balance(self.quote_token), 1) + + # Reset the logs + self.event_logger.clear() + + # Try to sell back the same amount to the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.98") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + order_id = self._place_order(False, amount, OrderType.LIMIT, price, 2) + order_completed_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCompletedEvent)) + trade_events = [t for t in self.event_logger.event_log if isinstance(t, OrderFilledEvent)] + base_amount_traded = sum(t.amount for t in trade_events) + quote_amount_traded = sum(t.amount * t.price for t in trade_events) + + self.assertTrue([evt.order_type == OrderType.LIMIT for evt in trade_events]) + self.assertEqual(order_id, order_completed_event.order_id) + self.assertEqual(amount, order_completed_event.base_asset_amount) + self.assertEqual("BTC", order_completed_event.base_asset) + self.assertEqual("USD", order_completed_event.quote_asset) + self.assertAlmostEqual(base_amount_traded, order_completed_event.base_asset_amount) + self.assertAlmostEqual(quote_amount_traded, order_completed_event.quote_asset_amount) + self.assertGreater(order_completed_event.fee_amount, Decimal(0)) + self.assertTrue(any([isinstance(event, SellOrderCreatedEvent) and event.order_id == order_id + for event in self.event_logger.event_log])) + + # check available base balance gets updated, we need to wait a bit for the balance message to arrive + expected_base_bal = base_bal + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.ev_loop.run_until_complete(self.connector._update_balances()) + self.ev_loop.run_until_complete(asyncio.sleep(5)) + self.assertAlmostEqual(expected_base_bal, self.connector.get_available_balance(self.base_token), 5) + + def test_limit_makers_unfilled(self): + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.8") + price = self.connector.quantize_order_price(self.trading_pair, price) + price_quantum = self.connector.get_order_price_quantum(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.ev_loop.run_until_complete(self.connector._update_balances()) + self.ev_loop.run_until_complete(asyncio.sleep(2)) + quote_bal = self.connector.get_available_balance(self.quote_token) + + cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + # check available quote balance gets updated, we need to wait a bit for the balance message to arrive + taker_fee = self.connector.estimate_fee_pct(False) + quote_amount = (math.ceil(((price * amount) * (Decimal("1") + taker_fee)) / price_quantum) * price_quantum) + expected_quote_bal = quote_bal - quote_amount + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.ev_loop.run_until_complete(self.connector._update_balances()) + self.ev_loop.run_until_complete(asyncio.sleep(2)) + + self.assertAlmostEqual(expected_quote_bal, self.connector.get_available_balance(self.quote_token), 5) + self._cancel_order(cl_order_id) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.2") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + cl_order_id = self._place_order(False, amount, OrderType.LIMIT_MAKER, price, 2) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + self._cancel_order(cl_order_id) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + # # @TODO: find a way to create "rejected" + # def test_limit_maker_rejections(self): + # price = self.connector.get_price(self.trading_pair, True) * Decimal("1.2") + # price = self.connector.quantize_order_price(self.trading_pair, price) + # amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.000001")) + # cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1) + # event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + # self.assertEqual(cl_order_id, event.order_id) + + # price = self.connector.get_price(self.trading_pair, False) * Decimal("0.8") + # price = self.connector.quantize_order_price(self.trading_pair, price) + # amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.000001")) + # cl_order_id = self._place_order(False, amount, OrderType.LIMIT_MAKER, price, 2) + # event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + # self.assertEqual(cl_order_id, event.order_id) + + def test_cancel_all(self): + bid_price = self.connector.get_price(self.trading_pair, True) + ask_price = self.connector.get_price(self.trading_pair, False) + bid_price = self.connector.quantize_order_price(self.trading_pair, bid_price * Decimal("0.9")) + ask_price = self.connector.quantize_order_price(self.trading_pair, ask_price * Decimal("1.1")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + buy_id = self._place_order(True, amount, OrderType.LIMIT, bid_price, 1) + sell_id = self._place_order(False, amount, OrderType.LIMIT, ask_price, 2) + + self.ev_loop.run_until_complete(asyncio.sleep(1)) + asyncio.ensure_future(self.connector.cancel_all(15)) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + cancel_events = [t for t in self.event_logger.event_log if isinstance(t, OrderCancelledEvent)] + self.assertEqual({buy_id, sell_id}, {o.order_id for o in cancel_events}) + + def test_order_quantized_values(self): + bid_price: Decimal = self.connector.get_price(self.trading_pair, True) + ask_price: Decimal = self.connector.get_price(self.trading_pair, False) + mid_price: Decimal = (bid_price + ask_price) / 2 + + # Make sure there's enough balance to make the limit orders. + self.assertGreater(self.connector.get_balance("BTC"), Decimal("0.0005")) + self.assertGreater(self.connector.get_balance("USD"), Decimal("10")) + + # Intentionally set some prices with too many decimal places s.t. they + # need to be quantized. Also, place them far away from the mid-price s.t. they won't + # get filled during the test. + bid_price = self.connector.quantize_order_price(self.trading_pair, mid_price * Decimal("0.9333192292111341")) + ask_price = self.connector.quantize_order_price(self.trading_pair, mid_price * Decimal("1.1492431474884933")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.000123456")) + + # Test bid order + cl_order_id_1 = self._place_order(True, amount, OrderType.LIMIT, bid_price, 1) + # Wait for the order created event and examine the order made + self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + + # Test ask order + cl_order_id_2 = self._place_order(False, amount, OrderType.LIMIT, ask_price, 1) + # Wait for the order created event and examine and order made + self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCreatedEvent)) + + self._cancel_order(cl_order_id_1) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self._cancel_order(cl_order_id_2) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + + def test_orders_saving_and_restoration(self): + config_path = "test_config" + strategy_name = "test_strategy" + sql = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, db_path=self.db_path) + order_id = None + recorder = MarketsRecorder(sql, [self.connector], config_path, strategy_name) + recorder.start() + + try: + self.connector._in_flight_orders.clear() + self.assertEqual(0, len(self.connector.tracking_states)) + + # Try to put limit buy order for 0.02 ETH worth of ZRX, and watch for order creation event. + current_bid_price: Decimal = self.connector.get_price(self.trading_pair, True) + price: Decimal = current_bid_price * Decimal("0.8") + price = self.connector.quantize_order_price(self.trading_pair, price) + + amount: Decimal = Decimal("0.0001") + amount = self.connector.quantize_order_amount(self.trading_pair, amount) + + cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + + # Verify tracking states + self.assertEqual(1, len(self.connector.tracking_states)) + self.assertEqual(cl_order_id, list(self.connector.tracking_states.keys())[0]) + + # Verify orders from recorder + recorded_orders: List[Order] = recorder.get_orders_for_config_and_market(config_path, self.connector) + self.assertEqual(1, len(recorded_orders)) + self.assertEqual(cl_order_id, recorded_orders[0].id) + + # Verify saved market states + saved_market_states: MarketState = recorder.get_market_states(config_path, self.connector) + self.assertIsNotNone(saved_market_states) + self.assertIsInstance(saved_market_states.saved_state, dict) + self.assertGreater(len(saved_market_states.saved_state), 0) + + # Close out the current market and start another market. + self.connector.stop(self._clock) + self.ev_loop.run_until_complete(asyncio.sleep(5)) + self.clock.remove_iterator(self.connector) + for event_tag in self.events: + self.connector.remove_listener(event_tag, self.event_logger) + # Clear the event loop + self.event_logger.clear() + new_connector = CoinzoomExchange(API_KEY, API_SECRET, API_USERNAME, [self.trading_pair], True) + for event_tag in self.events: + new_connector.add_listener(event_tag, self.event_logger) + recorder.stop() + recorder = MarketsRecorder(sql, [new_connector], config_path, strategy_name) + recorder.start() + saved_market_states = recorder.get_market_states(config_path, new_connector) + self.clock.add_iterator(new_connector) + self.ev_loop.run_until_complete(self.wait_til_ready(new_connector)) + self.assertEqual(0, len(new_connector.limit_orders)) + self.assertEqual(0, len(new_connector.tracking_states)) + new_connector.restore_tracking_states(saved_market_states.saved_state) + self.assertEqual(1, len(new_connector.limit_orders)) + self.assertEqual(1, len(new_connector.tracking_states)) + + # Cancel the order and verify that the change is saved. + self._cancel_order(cl_order_id, new_connector) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + recorder.save_market_states(config_path, new_connector) + order_id = None + self.assertEqual(0, len(new_connector.limit_orders)) + self.assertEqual(0, len(new_connector.tracking_states)) + saved_market_states = recorder.get_market_states(config_path, new_connector) + self.assertEqual(0, len(saved_market_states.saved_state)) + finally: + if order_id is not None: + self.connector.cancel(self.trading_pair, cl_order_id) + self.run_parallel(self.event_logger.wait_for(OrderCancelledEvent)) + + recorder.stop() + os.unlink(self.db_path) + + def test_update_last_prices(self): + # This is basic test to see if order_book last_trade_price is initiated and updated. + for order_book in self.connector.order_books.values(): + for _ in range(5): + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.assertFalse(math.isnan(order_book.last_trade_price)) + + def test_filled_orders_recorded(self): + config_path: str = "test_config" + strategy_name: str = "test_strategy" + sql = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, db_path=self.db_path) + order_id = None + recorder = MarketsRecorder(sql, [self.connector], config_path, strategy_name) + recorder.start() + + try: + # Try to buy some token from the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.05") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + order_id = self._place_order(True, amount, OrderType.LIMIT, price, 1) + self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + + # Reset the logs + self.event_logger.clear() + + # Try to sell back the same amount to the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.95") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + order_id = self._place_order(False, amount, OrderType.LIMIT, price, 2) + self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + + # Query the persisted trade logs + trade_fills: List[TradeFill] = recorder.get_trades_for_config(config_path) + self.assertGreaterEqual(len(trade_fills), 2) + buy_fills: List[TradeFill] = [t for t in trade_fills if t.trade_type == "BUY"] + sell_fills: List[TradeFill] = [t for t in trade_fills if t.trade_type == "SELL"] + self.assertGreaterEqual(len(buy_fills), 1) + self.assertGreaterEqual(len(sell_fills), 1) + + order_id = None + + finally: + if order_id is not None: + self.connector.cancel(self.trading_pair, order_id) + self.run_parallel(self.event_logger.wait_for(OrderCancelledEvent)) + + recorder.stop() + os.unlink(self.db_path) diff --git a/test/connector/exchange/coinzoom/test_coinzoom_order_book_tracker.py b/test/connector/exchange/coinzoom/test_coinzoom_order_book_tracker.py new file mode 100755 index 0000000000..62a1a1b6d9 --- /dev/null +++ b/test/connector/exchange/coinzoom/test_coinzoom_order_book_tracker.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +import sys +import math +import time +import asyncio +import logging +import unittest +from async_timeout import timeout +from os.path import join, realpath +from typing import Dict, Optional, List +from hummingbot.core.event.event_logger import EventLogger +from hummingbot.core.event.events import OrderBookEvent, OrderBookTradeEvent, TradeType +from hummingbot.connector.exchange.coinzoom.coinzoom_order_book_tracker import CoinzoomOrderBookTracker +from hummingbot.connector.exchange.coinzoom.coinzoom_api_order_book_data_source import CoinzoomAPIOrderBookDataSource +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL + + +sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +logging.basicConfig(level=METRICS_LOG_LEVEL) + + +class CoinzoomOrderBookTrackerUnitTest(unittest.TestCase): + order_book_tracker: Optional[CoinzoomOrderBookTracker] = None + events: List[OrderBookEvent] = [ + OrderBookEvent.TradeEvent + ] + trading_pairs: List[str] = [ + "BTC-USD", + "ETH-USD", + ] + + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + cls.order_book_tracker: CoinzoomOrderBookTracker = CoinzoomOrderBookTracker(cls.trading_pairs) + cls.order_book_tracker.start() + cls.ev_loop.run_until_complete(cls.wait_til_tracker_ready()) + + @classmethod + async def wait_til_tracker_ready(cls): + async with timeout(20): + while True: + if len(cls.order_book_tracker.order_books) > 0: + print("Initialized real-time order books.") + return + await asyncio.sleep(1) + + async def run_parallel_async(self, *tasks, timeout=None): + future: asyncio.Future = asyncio.ensure_future(asyncio.gather(*tasks)) + timer = 0 + while not future.done(): + if timeout and timer > timeout: + raise Exception("Timeout running parallel async tasks in tests") + timer += 1 + now = time.time() + _next_iteration = now // 1.0 + 1 # noqa: F841 + await asyncio.sleep(1.0) + return future.result() + + def run_parallel(self, *tasks): + return self.ev_loop.run_until_complete(self.run_parallel_async(*tasks, timeout=60)) + + def setUp(self): + self.event_logger = EventLogger() + for event_tag in self.events: + for trading_pair, order_book in self.order_book_tracker.order_books.items(): + order_book.add_listener(event_tag, self.event_logger) + + def test_order_book_trade_event_emission(self): + """ + Tests if the order book tracker is able to retrieve order book trade message from exchange and emit order book + trade events after correctly parsing the trade messages + """ + self.run_parallel(self.event_logger.wait_for(OrderBookTradeEvent)) + for ob_trade_event in self.event_logger.event_log: + self.assertTrue(type(ob_trade_event) == OrderBookTradeEvent) + self.assertTrue(ob_trade_event.trading_pair in self.trading_pairs) + self.assertTrue(type(ob_trade_event.timestamp) in [float, int]) + self.assertTrue(type(ob_trade_event.amount) == float) + self.assertTrue(type(ob_trade_event.price) == float) + self.assertTrue(type(ob_trade_event.type) == TradeType) + # datetime is in milliseconds + self.assertTrue(math.ceil(math.log10(ob_trade_event.timestamp)) == 13) + self.assertTrue(ob_trade_event.amount > 0) + self.assertTrue(ob_trade_event.price > 0) + + def test_tracker_integrity(self): + # Wait 5 seconds to process some diffs. + self.ev_loop.run_until_complete(asyncio.sleep(5.0)) + order_books: Dict[str, OrderBook] = self.order_book_tracker.order_books + eth_usd: OrderBook = order_books["ETH-USD"] + self.assertIsNot(eth_usd.last_diff_uid, 0) + self.assertGreaterEqual(eth_usd.get_price_for_volume(True, 10).result_price, + eth_usd.get_price(True)) + self.assertLessEqual(eth_usd.get_price_for_volume(False, 10).result_price, + eth_usd.get_price(False)) + + def test_api_get_last_traded_prices(self): + prices = self.ev_loop.run_until_complete( + CoinzoomAPIOrderBookDataSource.get_last_traded_prices(["BTC-USD", "LTC-BTC"])) + for key, value in prices.items(): + print(f"{key} last_trade_price: {value}") + self.assertGreater(prices["BTC-USD"], 1000) + self.assertLess(prices["LTC-BTC"], 1) diff --git a/test/connector/exchange/coinzoom/test_coinzoom_user_stream_tracker.py b/test/connector/exchange/coinzoom/test_coinzoom_user_stream_tracker.py new file mode 100644 index 0000000000..f9f85f335d --- /dev/null +++ b/test/connector/exchange/coinzoom/test_coinzoom_user_stream_tracker.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +import sys +import asyncio +import logging +import unittest +import conf + +from os.path import join, realpath +from hummingbot.connector.exchange.coinzoom.coinzoom_user_stream_tracker import CoinzoomUserStreamTracker +from hummingbot.connector.exchange.coinzoom.coinzoom_auth import CoinzoomAuth +from hummingbot.core.utils.async_utils import safe_ensure_future +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL + + +sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +logging.basicConfig(level=METRICS_LOG_LEVEL) + + +class CoinzoomUserStreamTrackerUnitTest(unittest.TestCase): + api_key = conf.coinzoom_api_key + api_secret = conf.coinzoom_secret_key + api_username = conf.coinzoom_username + + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + cls.trading_pairs = ["BTC-USD"] + cls.user_stream_tracker: CoinzoomUserStreamTracker = CoinzoomUserStreamTracker( + coinzoom_auth=CoinzoomAuth(cls.api_key, cls.api_secret, cls.api_username), + trading_pairs=cls.trading_pairs) + cls.user_stream_tracker_task: asyncio.Task = safe_ensure_future(cls.user_stream_tracker.start()) + + def test_user_stream(self): + # Wait process some msgs. + print("Sleeping for 30s to gather some user stream messages.") + self.ev_loop.run_until_complete(asyncio.sleep(30.0)) + print(self.user_stream_tracker.user_stream) diff --git a/test/connector/exchange/digifinex/__init__.py b/test/connector/exchange/digifinex/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/connector/exchange/digifinex/fixture.py b/test/connector/exchange/digifinex/fixture.py new file mode 100644 index 0000000000..85390bb01f --- /dev/null +++ b/test/connector/exchange/digifinex/fixture.py @@ -0,0 +1,127 @@ +BALANCES = { + 'id': 815129178419638016, 'method': 'private/get-account-summary', 'code': 0, + 'result': { + 'accounts': [{'balance': 50, 'available': 50, 'order': 0.0, 'stake': 0, 'currency': 'USDT'}, + {'balance': 0.002, 'available': 0.002, 'order': 0, 'stake': 0, 'currency': 'BTC'}] + } +} + +INSTRUMENTS = {'id': -1, 'method': 'public/get-instruments', 'code': 0, 'result': {'instruments': [ + {'instrument_name': 'BTC_USDT', 'quote_currency': 'USDT', 'base_currency': 'BTC', 'price_decimals': 2, + 'quantity_decimals': 6}, + {'instrument_name': 'ETH_USDT', 'quote_currency': 'USDT', 'base_currency': 'ETH', 'price_decimals': 2, + 'quantity_decimals': 5}, +]}} + +TICKERS = {'code': 0, 'method': 'public/get-ticker', 'result': { + 'instrument_name': 'BTC_USDT', + 'data': [{'i': 'BTC_USDT', 'b': 11490.0, 'k': 11492.05, + 'a': 11490.0, 't': 1598674849297, + 'v': 754.531926, 'h': 11546.11, 'l': 11366.62, + 'c': 104.19}]}} + +GET_BOOK = { + "code": 0, "method": "public/get-book", "result": { + "instrument_name": "BTC_USDT", "depth": 5, "data": + [{"bids": [[11490.00, 0.010676, 1], [11488.34, 0.055374, 1], [11487.47, 0.003000, 1], + [11486.50, 0.031032, 1], + [11485.97, 0.087074, 1]], + "asks": [[11492.05, 0.232044, 1], [11492.06, 0.497900, 1], [11493.12, 2.005693, 1], + [11494.12, 7.000000, 1], + [11494.41, 0.032853, 1]], "t": 1598676097390}]}} + +PLACE_ORDER = {'id': 632194937848317440, 'method': 'private/create-order', 'code': 0, + 'result': {'order_id': '1', 'client_oid': 'buy-BTC-USDT-1598607082008742'}} + +CANCEL = {'id': 31484728768575776, 'method': 'private/cancel-order', 'code': 0} + +UNFILLED_ORDER = { + 'id': 798015906490506624, + 'method': 'private/get-order-detail', + 'code': 0, + 'result': { + 'trade_list': [], + 'order_info': { + 'status': 'ACTIVE', + 'side': 'BUY', + 'price': 9164.82, + 'quantity': 0.0001, + 'order_id': '1', + 'client_oid': 'buy-BTC-USDT-1598607082008742', + 'create_time': 1598607082329, + 'update_time': 1598607082332, + 'type': 'LIMIT', + 'instrument_name': 'BTC_USDT', + 'avg_price': 0.0, + 'cumulative_quantity': 0.0, + 'cumulative_value': 0.0, + 'fee_currency': 'BTC', + 'exec_inst': 'POST_ONLY', + 'time_in_force': 'GOOD_TILL_CANCEL'} + } +} + +WS_INITIATED = {'id': 317343764453238848, 'method': 'public/auth', 'code': 0} +WS_SUBSCRIBE = {'id': 802984382214439040, 'method': 'subscribe', 'code': 0} +WS_HEARTBEAT = {'id': 1598755526207, 'method': 'public/heartbeat'} + +WS_ORDER_FILLED = { + 'id': -1, 'method': 'subscribe', 'code': 0, + 'result': { + 'instrument_name': 'BTC_USDT', + 'subscription': 'user.order.BTC_USDT', + 'channel': 'user.order', + 'data': [ + {'status': 'FILLED', + 'side': 'BUY', + 'price': 12080.9, + 'quantity': 0.0001, + 'order_id': '1', + 'client_oid': 'buy-BTC-USDT-1598681216010994', + 'create_time': 1598681216332, + 'update_time': 1598681216334, + 'type': 'LIMIT', + 'instrument_name': 'BTC_USDT', + 'avg_price': 11505.62, + 'cumulative_quantity': 0.0001, + 'cumulative_value': 11.50562, + 'fee_currency': 'BTC', + 'exec_inst': '', + 'time_in_force': 'GOOD_TILL_CANCEL'}]}} + +WS_TRADE = { + 'id': -1, 'method': 'subscribe', 'code': 0, + 'result': { + 'instrument_name': 'BTC_USDT', + 'subscription': 'user.trade.BTC_USDT', + 'channel': 'user.trade', + 'data': [ + {'side': 'BUY', + 'fee': 1.6e-06, + 'trade_id': '699422550491763776', + 'instrument_name': 'BTC_USDT', + 'create_time': 1598681216334, + 'traded_price': 11505.62, + 'traded_quantity': 0.0001, + 'fee_currency': 'BTC', + 'order_id': '1'}]}} + +WS_BALANCE = { + 'id': -1, 'method': 'subscribe', 'code': 0, + 'result': { + 'subscription': 'user.balance', 'channel': 'user.balance', + 'data': [{'balance': 47, 'available': 46, + 'order': 1, 'stake': 0, + 'currency': 'USDT'}]}} + +WS_ORDER_CANCELLED = { + 'id': -1, 'method': 'subscribe', 'code': 0, + 'result': { + 'instrument_name': 'BTC_USDT', 'subscription': 'user.order.BTC_USDT', + 'channel': 'user.order', 'data': [ + {'status': 'CANCELED', 'side': 'BUY', 'price': 13918.12, 'quantity': 0.0001, + 'order_id': '1', 'client_oid': 'buy-BTC-USDT-1598757896008300', + 'create_time': 1598757896312, 'update_time': 1598757896312, 'type': 'LIMIT', + 'instrument_name': 'BTC_USDT', 'avg_price': 0.0, 'cumulative_quantity': 0.0, + 'cumulative_value': 0.0, 'fee_currency': 'BTC', 'exec_inst': 'POST_ONLY', + 'time_in_force': 'GOOD_TILL_CANCEL'}]}} diff --git a/test/connector/exchange/digifinex/test_digifinex_auth.py b/test/connector/exchange/digifinex/test_digifinex_auth.py new file mode 100644 index 0000000000..4fcbc59c13 --- /dev/null +++ b/test/connector/exchange/digifinex/test_digifinex_auth.py @@ -0,0 +1,32 @@ +from os.path import join, realpath +import sys; sys.path.insert(0, realpath(join(__file__, "../../../../../"))) + +import asyncio +import unittest + +import conf +from hummingbot.connector.exchange.digifinex.digifinex_auth import DigifinexAuth +from hummingbot.connector.exchange.digifinex.digifinex_websocket import DigifinexWebsocket + + +class TestAuth(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + api_key = conf.digifinex_api_key + secret_key = conf.digifinex_secret_key + cls.auth = DigifinexAuth(api_key, secret_key) + cls.ws = DigifinexWebsocket(cls.auth) + + async def ws_auth(self): + await self.ws.connect() + await self.ws.subscribe("balance", ["USDT", "BTC", "ETH"]) + + # no msg will arrive until balance changed after subscription + # async for response in self.ws.on_message(): + # if (response.get("method") == "subscribe"): + # return response + + def test_ws_auth(self): + self.ev_loop.run_until_complete(self.ws_auth()) + # assert result["code"] == 0 diff --git a/test/connector/exchange/digifinex/test_digifinex_exchange.py b/test/connector/exchange/digifinex/test_digifinex_exchange.py new file mode 100644 index 0000000000..54f9be5c57 --- /dev/null +++ b/test/connector/exchange/digifinex/test_digifinex_exchange.py @@ -0,0 +1,539 @@ +# print('__file__={0:<35} | __name__={1:<20} | __package__={2:<20}'.format(__file__,__name__,str(__package__))) +import os +from os.path import join, realpath +import sys; sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +import asyncio +import logging +from decimal import Decimal +import unittest +import contextlib +import time +from typing import List +# from unittest import mock +import conf +import math + +from test.connector.exchange.digifinex import fixture +from hummingbot.core.clock import Clock, ClockMode +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL +from hummingbot.core.utils.async_utils import safe_gather, safe_ensure_future +from hummingbot.core.event.event_logger import EventLogger +from hummingbot.core.event.events import ( + BuyOrderCompletedEvent, + BuyOrderCreatedEvent, + MarketEvent, + OrderFilledEvent, + OrderType, + SellOrderCompletedEvent, + SellOrderCreatedEvent, + OrderCancelledEvent +) +from hummingbot.model.sql_connection_manager import ( + SQLConnectionManager, + SQLConnectionType +) +from hummingbot.model.market_state import MarketState +from hummingbot.model.order import Order +from hummingbot.model.trade_fill import TradeFill +from hummingbot.connector.markets_recorder import MarketsRecorder +from hummingbot.connector.exchange.digifinex.digifinex_exchange import DigifinexExchange +# from hummingbot.connector.exchange.digifinex.digifinex_constants import WSS_PUBLIC_URL, WSS_PRIVATE_URL +# from test.integration.humming_web_app import HummingWebApp +# from test.integration.humming_ws_server import HummingWsServerFactory + +# API_MOCK_ENABLED = conf.mock_api_enabled is not None and conf.mock_api_enabled.lower() in ['true', 'yes', '1'] +API_MOCK_ENABLED = False + +logging.basicConfig(level=METRICS_LOG_LEVEL) +# logging.basicConfig(level=logging.NETWORK) +# logging.basicConfig(level=logging.DEBUG) +API_KEY = conf.digifinex_api_key +API_SECRET = conf.digifinex_secret_key +# BASE_API_URL = "openapi.digifinex.com" + + +class DigifinexExchangeUnitTest(unittest.TestCase): + events: List[MarketEvent] = [ + MarketEvent.BuyOrderCompleted, + MarketEvent.SellOrderCompleted, + MarketEvent.OrderFilled, + MarketEvent.TransactionFailure, + MarketEvent.BuyOrderCreated, + MarketEvent.SellOrderCreated, + MarketEvent.OrderCancelled, + MarketEvent.OrderFailure + ] + connector: DigifinexExchange + event_logger: EventLogger + trading_pair = "BTC-USDT" + base_token, quote_token = trading_pair.split("-") + stack: contextlib.ExitStack + sql: SQLConnectionManager + + @classmethod + def setUpClass(cls): + global MAINNET_RPC_URL + + cls.ev_loop = asyncio.get_event_loop() + + if API_MOCK_ENABLED: + raise NotImplementedError() + # cls.web_app = HummingWebApp.get_instance() + # cls.web_app.add_host_to_mock(BASE_API_URL, []) + # cls.web_app.start() + # cls.ev_loop.run_until_complete(cls.web_app.wait_til_started()) + # cls._patcher = mock.patch("aiohttp.client.URL") + # cls._url_mock = cls._patcher.start() + # cls._url_mock.side_effect = cls.web_app.reroute_local + # cls.web_app.update_response("get", BASE_API_URL, "/v2/public/get-ticker", fixture.TICKERS) + # cls.web_app.update_response("get", BASE_API_URL, "/v2/public/get-instruments", fixture.INSTRUMENTS) + # cls.web_app.update_response("get", BASE_API_URL, "/v2/public/get-book", fixture.GET_BOOK) + # cls.web_app.update_response("post", BASE_API_URL, "/v2/private/get-account-summary", fixture.BALANCES) + # cls.web_app.update_response("post", BASE_API_URL, "/v2/private/cancel-order", fixture.CANCEL) + + # HummingWsServerFactory.start_new_server(WSS_PRIVATE_URL) + # HummingWsServerFactory.start_new_server(WSS_PUBLIC_URL) + # cls._ws_patcher = unittest.mock.patch("websockets.connect", autospec=True) + # cls._ws_mock = cls._ws_patcher.start() + # cls._ws_mock.side_effect = HummingWsServerFactory.reroute_ws_connect + + cls.clock: Clock = Clock(ClockMode.REALTIME) + cls.connector: DigifinexExchange = DigifinexExchange( + digifinex_api_key=API_KEY, + digifinex_secret_key=API_SECRET, + trading_pairs=[cls.trading_pair], + trading_required=True + ) + print("Initializing Digifinex market... this will take about a minute.") + cls.clock.add_iterator(cls.connector) + cls.stack: contextlib.ExitStack = contextlib.ExitStack() + cls._clock = cls.stack.enter_context(cls.clock) + # if API_MOCK_ENABLED: + # HummingWsServerFactory.send_json_threadsafe(WSS_PRIVATE_URL, fixture.WS_INITIATED, delay=0.5) + # HummingWsServerFactory.send_json_threadsafe(WSS_PRIVATE_URL, fixture.WS_SUBSCRIBE, delay=0.51) + # HummingWsServerFactory.send_json_threadsafe(WSS_PRIVATE_URL, fixture.WS_HEARTBEAT, delay=0.52) + + cls.ev_loop.run_until_complete(cls.wait_til_ready()) + print("Ready.") + + @classmethod + def tearDownClass(cls) -> None: + cls.stack.close() + # if API_MOCK_ENABLED: + # cls.web_app.stop() + # cls._patcher.stop() + # cls._ws_patcher.stop() + + @classmethod + async def wait_til_ready(cls, connector = None): + if connector is None: + connector = cls.connector + while True: + now = time.time() + next_iteration = now // 1.0 + 1 + if connector.ready: + break + else: + await cls._clock.run_til(next_iteration) + await asyncio.sleep(1.0) + + def setUp(self): + self.db_path: str = realpath(join(__file__, "../connector_test.sqlite")) + try: + # on windows cannot unlink the sqlite db file before closing the db + if os.name != 'nt': + os.unlink(self.db_path) + except FileNotFoundError: + pass + + self.event_logger = EventLogger() + for event_tag in self.events: + self.connector.add_listener(event_tag, self.event_logger) + + def tearDown(self): + for event_tag in self.events: + self.connector.remove_listener(event_tag, self.event_logger) + self.event_logger = None + # self.sql._engine.dispose() + + async def run_parallel_async(self, *tasks): + future: asyncio.Future = safe_ensure_future(safe_gather(*tasks)) + while not future.done(): + now = time.time() + next_iteration = now // 1.0 + 1 + await self._clock.run_til(next_iteration) + await asyncio.sleep(1.0) + return future.result() + + def run_parallel(self, *tasks): + return self.ev_loop.run_until_complete(self.run_parallel_async(*tasks)) + + def test_estimate_fee(self): + maker_fee = self.connector.estimate_fee_pct(True) + self.assertAlmostEqual(maker_fee, Decimal("0.001")) + taker_fee = self.connector.estimate_fee_pct(False) + self.assertAlmostEqual(taker_fee, Decimal("0.001")) + + def _place_order(self, is_buy, amount, order_type, price, ex_order_id, get_order_fixture=None, + ws_trade_fixture=None, ws_order_fixture=None) -> str: + # if API_MOCK_ENABLED: + # data = fixture.PLACE_ORDER.copy() + # data["result"]["order_id"] = str(ex_order_id) + # self.web_app.update_response("post", BASE_API_URL, "/v2/private/create-order", data) + if is_buy: + cl_order_id = self.connector.buy(self.trading_pair, amount, order_type, price) + else: + cl_order_id = self.connector.sell(self.trading_pair, amount, order_type, price) + # if API_MOCK_ENABLED: + # if get_order_fixture is not None: + # data = get_order_fixture.copy() + # data["result"]["order_info"]["client_oid"] = cl_order_id + # data["result"]["order_info"]["order_id"] = ex_order_id + # self.web_app.update_response("post", BASE_API_URL, "/v2/private/get-order-detail", data) + # if ws_trade_fixture is not None: + # data = ws_trade_fixture.copy() + # data["result"]["data"][0]["order_id"] = str(ex_order_id) + # HummingWsServerFactory.send_json_threadsafe(WSS_PRIVATE_URL, data, delay=0.1) + # if ws_order_fixture is not None: + # data = ws_order_fixture.copy() + # data["result"]["data"][0]["order_id"] = str(ex_order_id) + # data["result"]["data"][0]["client_oid"] = cl_order_id + # HummingWsServerFactory.send_json_threadsafe(WSS_PRIVATE_URL, data, delay=0.12) + return cl_order_id + + def _cancel_order(self, cl_order_id): + self.connector.cancel(self.trading_pair, cl_order_id) + # if API_MOCK_ENABLED: + # data = fixture.WS_ORDER_CANCELLED.copy() + # data["result"]["data"][0]["client_oid"] = cl_order_id + # HummingWsServerFactory.send_json_threadsafe(WSS_PRIVATE_URL, data, delay=0.1) + + def test_buy_and_sell(self): + self.ev_loop.run_until_complete(self.connector.cancel_all(0)) + + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.05") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + quote_bal = self.connector.get_available_balance(self.quote_token) + base_bal = self.connector.get_available_balance(self.base_token) + + order_id = self._place_order(True, amount, OrderType.LIMIT, price, 1, None, + fixture.WS_TRADE) + order_completed_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(2)) + trade_events = [t for t in self.event_logger.event_log if isinstance(t, OrderFilledEvent)] + base_amount_traded = sum(t.amount for t in trade_events) + quote_amount_traded = sum(t.amount * t.price for t in trade_events) + + self.assertTrue([evt.order_type == OrderType.LIMIT for evt in trade_events]) + self.assertEqual(order_id, order_completed_event.order_id) + self.assertEqual(amount, order_completed_event.base_asset_amount) + self.assertEqual("BTC", order_completed_event.base_asset) + self.assertEqual("USDT", order_completed_event.quote_asset) + self.assertAlmostEqual(base_amount_traded, order_completed_event.base_asset_amount) + self.assertAlmostEqual(quote_amount_traded, order_completed_event.quote_asset_amount) + # todo: get fee + # self.assertGreater(order_completed_event.fee_amount, Decimal(0)) + self.assertTrue(any([isinstance(event, BuyOrderCreatedEvent) and event.order_id == order_id + for event in self.event_logger.event_log])) + + # check available quote balance gets updated, we need to wait a bit for the balance message to arrive + expected_quote_bal = quote_bal - quote_amount_traded + # self._mock_ws_bal_update(self.quote_token, expected_quote_bal) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.assertAlmostEqual(expected_quote_bal, self.connector.get_available_balance(self.quote_token), delta=0.1) + + # Reset the logs + self.event_logger.clear() + + # Try to sell back the same amount to the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.95") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + order_id = self._place_order(False, amount, OrderType.LIMIT, price, 2, None, + fixture.WS_TRADE) + order_completed_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCompletedEvent)) + trade_events = [t for t in self.event_logger.event_log if isinstance(t, OrderFilledEvent)] + base_amount_traded = sum(t.amount for t in trade_events) + quote_amount_traded = sum(t.amount * t.price for t in trade_events) + + self.assertTrue([evt.order_type == OrderType.LIMIT for evt in trade_events]) + self.assertEqual(order_id, order_completed_event.order_id) + self.assertEqual(amount, order_completed_event.base_asset_amount) + self.assertEqual("BTC", order_completed_event.base_asset) + self.assertEqual("USDT", order_completed_event.quote_asset) + self.assertAlmostEqual(base_amount_traded, order_completed_event.base_asset_amount) + self.assertAlmostEqual(quote_amount_traded, order_completed_event.quote_asset_amount) + # todo: get fee + # self.assertGreater(order_completed_event.fee_amount, Decimal(0)) + self.assertTrue(any([isinstance(event, SellOrderCreatedEvent) and event.order_id == order_id + for event in self.event_logger.event_log])) + + # check available base balance gets updated, we need to wait a bit for the balance message to arrive + expected_base_bal = base_bal + # self._mock_ws_bal_update(self.base_token, expected_base_bal) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.assertAlmostEqual(expected_base_bal, self.connector.get_available_balance(self.base_token), 5) + + def test_limit_makers_unfilled(self): + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.8") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.00005")) + quote_bal = self.connector.get_available_balance(self.quote_token) + + # order_id = self.connector.buy(self.trading_pair, amount, OrderType.LIMIT_MAKER, price) + cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1, fixture.UNFILLED_ORDER) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + # check available quote balance gets updated, we need to wait a bit for the balance message to arrive + expected_quote_bal = quote_bal - (price * amount) + # self._mock_ws_bal_update(self.quote_token, expected_quote_bal) + self.ev_loop.run_until_complete(asyncio.sleep(2)) + self.assertAlmostEqual(expected_quote_bal, self.connector.get_available_balance(self.quote_token), 1) + self._cancel_order(cl_order_id) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.2") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + cl_order_id = self._place_order(False, amount, OrderType.LIMIT_MAKER, price, 2, fixture.UNFILLED_ORDER) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + self._cancel_order(cl_order_id) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + # def _mock_ws_bal_update(self, token, available): + # if API_MOCK_ENABLED: + # available = float(available) + # data = fixture.WS_BALANCE.copy() + # data["result"]["data"][0]["currency"] = token + # data["result"]["data"][0]["available"] = available + # HummingWsServerFactory.send_json_threadsafe(WSS_PRIVATE_URL, fixture.WS_BALANCE, delay=0.1) + + def test_limit_maker_rejections(self): + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.2") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1, None, None, + fixture.WS_ORDER_CANCELLED) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + price = self.connector.get_price(self.trading_pair, False) * Decimal("0.8") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + cl_order_id = self._place_order(False, amount, OrderType.LIMIT_MAKER, price, 2, None, None, + fixture.WS_ORDER_CANCELLED) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + def test_cancel_all(self): + bid_price = self.connector.get_price(self.trading_pair, True) + ask_price = self.connector.get_price(self.trading_pair, False) + bid_price = self.connector.quantize_order_price(self.trading_pair, bid_price * Decimal("0.7")) + ask_price = self.connector.quantize_order_price(self.trading_pair, ask_price * Decimal("1.5")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + buy_id = self._place_order(True, amount, OrderType.LIMIT, bid_price, 1) + sell_id = self._place_order(False, amount, OrderType.LIMIT, ask_price, 2) + + self.ev_loop.run_until_complete(asyncio.sleep(1)) + asyncio.ensure_future(self.connector.cancel_all(3)) + # if API_MOCK_ENABLED: + # data = fixture.WS_ORDER_CANCELLED.copy() + # data["result"]["data"][0]["client_oid"] = buy_id + # data["result"]["data"][0]["order_id"] = 1 + # HummingWsServerFactory.send_json_threadsafe(WSS_PRIVATE_URL, data, delay=0.1) + # self.ev_loop.run_until_complete(asyncio.sleep(1)) + # data = fixture.WS_ORDER_CANCELLED.copy() + # data["result"]["data"][0]["client_oid"] = sell_id + # data["result"]["data"][0]["order_id"] = 2 + # HummingWsServerFactory.send_json_threadsafe(WSS_PRIVATE_URL, data, delay=0.11) + self.ev_loop.run_until_complete(asyncio.sleep(3)) + cancel_events = [t for t in self.event_logger.event_log if isinstance(t, OrderCancelledEvent)] + self.assertEqual({buy_id, sell_id}, {o.order_id for o in cancel_events}) + + def test_order_price_precision(self): + bid_price: Decimal = self.connector.get_price(self.trading_pair, True) + ask_price: Decimal = self.connector.get_price(self.trading_pair, False) + mid_price: Decimal = (bid_price + ask_price) / 2 + amount: Decimal = Decimal("0.000123456") + + # Make sure there's enough balance to make the limit orders. + self.assertGreater(self.connector.get_balance("BTC"), Decimal("0.001")) + self.assertGreater(self.connector.get_balance("USDT"), Decimal("10")) + + # Intentionally set some prices with too many decimal places s.t. they + # need to be quantized. Also, place them far away from the mid-price s.t. they won't + # get filled during the test. + bid_price = mid_price * Decimal("0.9333192292111341") + ask_price = mid_price * Decimal("1.0492431474884933") + + cl_order_id_1 = self._place_order(True, amount, OrderType.LIMIT, bid_price, 1, fixture.UNFILLED_ORDER) + + # Wait for the order created event and examine the order made + self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + order = self.connector.in_flight_orders[cl_order_id_1] + quantized_bid_price = self.connector.quantize_order_price(self.trading_pair, bid_price) + quantized_bid_size = self.connector.quantize_order_amount(self.trading_pair, amount) + self.assertEqual(quantized_bid_price, order.price) + self.assertEqual(quantized_bid_size, order.amount) + + # Test ask order + cl_order_id_2 = self._place_order(False, amount, OrderType.LIMIT, ask_price, 1, fixture.UNFILLED_ORDER) + + # Wait for the order created event and examine and order made + self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCreatedEvent)) + order = self.connector.in_flight_orders[cl_order_id_2] + quantized_ask_price = self.connector.quantize_order_price(self.trading_pair, Decimal(ask_price)) + quantized_ask_size = self.connector.quantize_order_amount(self.trading_pair, Decimal(amount)) + self.assertEqual(quantized_ask_price, order.price) + self.assertEqual(quantized_ask_size, order.amount) + + self._cancel_order(cl_order_id_1) + self._cancel_order(cl_order_id_2) + + def test_orders_saving_and_restoration(self): + config_path = "test_config" + strategy_name = "test_strategy" + sql = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, db_path=self.db_path) + order_id = None + recorder = MarketsRecorder(sql, [self.connector], config_path, strategy_name) + recorder.start() + + try: + self.connector._in_flight_orders.clear() + self.assertEqual(0, len(self.connector.tracking_states)) + + # Try to put limit buy order for 0.02 ETH worth of ZRX, and watch for order creation event. + current_bid_price: Decimal = self.connector.get_price(self.trading_pair, True) + price: Decimal = current_bid_price * Decimal("0.8") + price = self.connector.quantize_order_price(self.trading_pair, price) + + amount: Decimal = Decimal("0.0001") + amount = self.connector.quantize_order_amount(self.trading_pair, amount) + + cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1, fixture.UNFILLED_ORDER) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + + # Verify tracking states + self.assertEqual(1, len(self.connector.tracking_states)) + self.assertEqual(cl_order_id, list(self.connector.tracking_states.keys())[0]) + + # Verify orders from recorder + recorded_orders: List[Order] = recorder.get_orders_for_config_and_market(config_path, self.connector) + self.assertEqual(1, len(recorded_orders)) + self.assertEqual(cl_order_id, recorded_orders[0].id) + + # Verify saved market states + saved_market_states: MarketState = recorder.get_market_states(config_path, self.connector) + self.assertIsNotNone(saved_market_states) + self.assertIsInstance(saved_market_states.saved_state, dict) + self.assertGreater(len(saved_market_states.saved_state), 0) + + # Close out the current market and start another market. + self.connector.stop(self._clock) + self.ev_loop.run_until_complete(asyncio.sleep(5)) + self.clock.remove_iterator(self.connector) + for event_tag in self.events: + self.connector.remove_listener(event_tag, self.event_logger) + new_connector = DigifinexExchange(API_KEY, API_SECRET, [self.trading_pair], True) + for event_tag in self.events: + new_connector.add_listener(event_tag, self.event_logger) + recorder.stop() + recorder = MarketsRecorder(sql, [new_connector], config_path, strategy_name) + recorder.start() + saved_market_states = recorder.get_market_states(config_path, new_connector) + self.clock.add_iterator(new_connector) + if not API_MOCK_ENABLED: + self.ev_loop.run_until_complete(self.wait_til_ready(new_connector)) + self.assertEqual(0, len(new_connector.limit_orders)) + self.assertEqual(0, len(new_connector.tracking_states)) + new_connector.restore_tracking_states(saved_market_states.saved_state) + self.assertEqual(1, len(new_connector.limit_orders)) + self.assertEqual(1, len(new_connector.tracking_states)) + + # Cancel the order and verify that the change is saved. + self._cancel_order(cl_order_id) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + order_id = None + self.assertEqual(0, len(new_connector.limit_orders)) + self.assertEqual(0, len(new_connector.tracking_states)) + saved_market_states = recorder.get_market_states(config_path, new_connector) + self.assertEqual(0, len(saved_market_states.saved_state)) + finally: + if order_id is not None: + self.connector.cancel(self.trading_pair, cl_order_id) + self.run_parallel(self.event_logger.wait_for(OrderCancelledEvent)) + + recorder.stop() + # sql._engine.dispose() + # on windows cannot unlink the sqlite db file before closing the db + if os.name != 'nt': + os.unlink(self.db_path) + + def test_update_last_prices(self): + # This is basic test to see if order_book last_trade_price is initiated and updated. + for order_book in self.connector.order_books.values(): + for _ in range(5): + self.ev_loop.run_until_complete(asyncio.sleep(1)) + print(order_book.last_trade_price) + self.assertFalse(math.isnan(order_book.last_trade_price)) + + def test_filled_orders_recorded(self): + config_path: str = "test_config" + strategy_name: str = "test_strategy" + sql = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, db_path=self.db_path) + order_id = None + recorder = MarketsRecorder(sql, [self.connector], config_path, strategy_name) + recorder.start() + + try: + # Try to buy some token from the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.05") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + order_id = self._place_order(True, amount, OrderType.LIMIT, price, 1, None, + fixture.WS_TRADE) + self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + + # Reset the logs + self.event_logger.clear() + + # Try to sell back the same amount to the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.95") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + order_id = self._place_order(False, amount, OrderType.LIMIT, price, 2, None, + fixture.WS_TRADE) + self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCompletedEvent)) + + # Query the persisted trade logs + trade_fills: List[TradeFill] = recorder.get_trades_for_config(config_path) + self.assertGreaterEqual(len(trade_fills), 2) + buy_fills: List[TradeFill] = [t for t in trade_fills if t.trade_type == "BUY"] + sell_fills: List[TradeFill] = [t for t in trade_fills if t.trade_type == "SELL"] + self.assertGreaterEqual(len(buy_fills), 1) + self.assertGreaterEqual(len(sell_fills), 1) + + order_id = None + + finally: + if order_id is not None: + self.connector.cancel(self.trading_pair, order_id) + self.run_parallel(self.event_logger.wait_for(OrderCancelledEvent)) + + recorder.stop() + # sql._engine.dispose() + # on windows cannot unlink the sqlite db file before closing the db + if os.name != 'nt': + os.unlink(self.db_path) + + +# unittest.main() diff --git a/test/connector/exchange/digifinex/test_digifinex_order_book_tracker.py b/test/connector/exchange/digifinex/test_digifinex_order_book_tracker.py new file mode 100644 index 0000000000..ebbe50fc0e --- /dev/null +++ b/test/connector/exchange/digifinex/test_digifinex_order_book_tracker.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +from os.path import join, realpath +import sys; sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +import math +import time +import asyncio +import logging +import unittest +from typing import Dict, Optional, List +from hummingbot.core.event.event_logger import EventLogger +from hummingbot.core.event.events import OrderBookEvent, OrderBookTradeEvent, TradeType +from hummingbot.connector.exchange.digifinex.digifinex_order_book_tracker import DigifinexOrderBookTracker +from hummingbot.connector.exchange.digifinex.digifinex_api_order_book_data_source import DigifinexAPIOrderBookDataSource +from hummingbot.core.data_type.order_book import OrderBook + + +class DigifinexOrderBookTrackerUnitTest(unittest.TestCase): + order_book_tracker: Optional[DigifinexOrderBookTracker] = None + events: List[OrderBookEvent] = [ + OrderBookEvent.TradeEvent + ] + trading_pairs: List[str] = [ + "BTC-USDT", + "ETH-USDT", + ] + + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + cls.order_book_tracker: DigifinexOrderBookTracker = DigifinexOrderBookTracker(cls.trading_pairs) + cls.order_book_tracker.start() + cls.ev_loop.run_until_complete(cls.wait_til_tracker_ready()) + + @classmethod + async def wait_til_tracker_ready(cls): + while True: + if len(cls.order_book_tracker.order_books) > 0: + print("Initialized real-time order books.") + return + await asyncio.sleep(1) + + async def run_parallel_async(self, *tasks, timeout=None): + logmsg = [f"{x.__qualname__}({x.cr_frame.f_locals if x.cr_frame is not None else None})" for x in tasks] + future: asyncio.Future = asyncio.ensure_future(asyncio.gather(*tasks)) + timer = 0 + while not future.done(): + if timeout and timer > timeout: + raise Exception("Timeout running parallel async tasks in tests") + timer += 1 + now = time.time() + _next_iteration = now // 1.0 + 1 # noqa: F841 + await asyncio.sleep(1.0) + if future._exception is not None: + logging.exception(logmsg) + return future.result() + + def run_parallel(self, *tasks): + return self.ev_loop.run_until_complete(self.run_parallel_async(*tasks)) + + def setUp(self): + self.event_logger = EventLogger() + for event_tag in self.events: + for trading_pair, order_book in self.order_book_tracker.order_books.items(): + order_book.add_listener(event_tag, self.event_logger) + + def test_order_book_trade_event_emission(self): + """ + Tests if the order book tracker is able to retrieve order book trade message from exchange and emit order book + trade events after correctly parsing the trade messages + """ + self.run_parallel(self.event_logger.wait_for(OrderBookTradeEvent)) + for ob_trade_event in self.event_logger.event_log: + self.assertTrue(type(ob_trade_event) == OrderBookTradeEvent) + self.assertTrue(ob_trade_event.trading_pair in self.trading_pairs) + self.assertTrue(type(ob_trade_event.timestamp) in [float, int]) + self.assertTrue(type(ob_trade_event.amount) == float) + self.assertTrue(type(ob_trade_event.price) == float) + self.assertTrue(type(ob_trade_event.type) == TradeType) + # datetime is in seconds + self.assertTrue(math.ceil(math.log10(ob_trade_event.timestamp)) == 10) + self.assertTrue(ob_trade_event.amount > 0) + self.assertTrue(ob_trade_event.price > 0) + + def test_tracker_integrity(self): + # Wait 5 seconds to process some diffs. + self.ev_loop.run_until_complete(asyncio.sleep(10.0)) + order_books: Dict[str, OrderBook] = self.order_book_tracker.order_books + eth_usdt: OrderBook = order_books["ETH-USDT"] + self.assertIsNot(eth_usdt.last_diff_uid, 0) + self.assertGreaterEqual(eth_usdt.get_price_for_volume(True, 10).result_price, + eth_usdt.get_price(True)) + self.assertLessEqual(eth_usdt.get_price_for_volume(False, 10).result_price, + eth_usdt.get_price(False)) + + def test_api_get_last_traded_prices(self): + prices = self.ev_loop.run_until_complete( + DigifinexAPIOrderBookDataSource.get_last_traded_prices(["BTC-USDT", "LTC-USDT"])) + for key, value in prices.items(): + print(f"{key} last_trade_price: {value}") + self.assertGreater(prices["BTC-USDT"], 1000) + self.assertLess(prices["LTC-USDT"], 10000) diff --git a/test/connector/exchange/digifinex/test_digifinex_user_stream_tracker.py b/test/connector/exchange/digifinex/test_digifinex_user_stream_tracker.py new file mode 100644 index 0000000000..f529ee22f6 --- /dev/null +++ b/test/connector/exchange/digifinex/test_digifinex_user_stream_tracker.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +from os.path import join, realpath +import sys; sys.path.insert(0, realpath(join(__file__, "../../../../../"))) + +import asyncio +import logging +import unittest +import conf + +from os.path import join, realpath +from hummingbot.connector.exchange.digifinex.digifinex_user_stream_tracker import DigifinexUserStreamTracker +from hummingbot.connector.exchange.digifinex.digifinex_global import DigifinexGlobal +from hummingbot.core.utils.async_utils import safe_ensure_future + +sys.path.insert(0, realpath(join(__file__, "../../../"))) + + +class DigifinexUserStreamTrackerUnitTest(unittest.TestCase): + api_key = conf.digifinex_api_key + api_secret = conf.digifinex_secret_key + + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + cls._global = DigifinexGlobal(cls.api_key, cls.api_secret) + cls.trading_pairs = ["BTC-USDT"] + cls.user_stream_tracker: DigifinexUserStreamTracker = DigifinexUserStreamTracker( + cls._global, trading_pairs=cls.trading_pairs) + cls.user_stream_tracker_task: asyncio.Task = safe_ensure_future(cls.user_stream_tracker.start()) + + def test_user_stream(self): + # Wait process some msgs. + self.ev_loop.run_until_complete(asyncio.sleep(120.0)) + print(self.user_stream_tracker.user_stream) + + +def main(): + logging.basicConfig(level=logging.INFO) + unittest.main() + + +if __name__ == "__main__": + main() diff --git a/test/connector/exchange/hitbtc/test_hitbtc_exchange.py b/test/connector/exchange/hitbtc/test_hitbtc_exchange.py index 0456f5a8a9..f63d4829d3 100644 --- a/test/connector/exchange/hitbtc/test_hitbtc_exchange.py +++ b/test/connector/exchange/hitbtc/test_hitbtc_exchange.py @@ -54,7 +54,7 @@ class HitbtcExchangeUnitTest(unittest.TestCase): ] connector: HitbtcExchange event_logger: EventLogger - trading_pair = "BTC-USD" + trading_pair = "BTC-USDT" base_token, quote_token = trading_pair.split("-") stack: contextlib.ExitStack @@ -144,7 +144,7 @@ def test_estimate_fee(self): def test_buy_and_sell(self): price = self.connector.get_price(self.trading_pair, True) * Decimal("1.02") price = self.connector.quantize_order_price(self.trading_pair, price) - amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0002")) quote_bal = self.connector.get_available_balance(self.quote_token) base_bal = self.connector.get_available_balance(self.base_token) @@ -159,7 +159,7 @@ def test_buy_and_sell(self): self.assertEqual(order_id, order_completed_event.order_id) self.assertEqual(amount, order_completed_event.base_asset_amount) self.assertEqual("BTC", order_completed_event.base_asset) - self.assertEqual("USD", order_completed_event.quote_asset) + self.assertEqual("USDT", order_completed_event.quote_asset) self.assertAlmostEqual(base_amount_traded, order_completed_event.base_asset_amount) self.assertAlmostEqual(quote_amount_traded, order_completed_event.quote_asset_amount) self.assertGreater(order_completed_event.fee_amount, Decimal(0)) @@ -178,7 +178,7 @@ def test_buy_and_sell(self): # Try to sell back the same amount to the exchange, and watch for completion event. price = self.connector.get_price(self.trading_pair, True) * Decimal("0.98") price = self.connector.quantize_order_price(self.trading_pair, price) - amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0002")) order_id = self._place_order(False, amount, OrderType.LIMIT, price, 2) order_completed_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCompletedEvent)) trade_events = [t for t in self.event_logger.event_log if isinstance(t, OrderFilledEvent)] @@ -189,7 +189,7 @@ def test_buy_and_sell(self): self.assertEqual(order_id, order_completed_event.order_id) self.assertEqual(amount, order_completed_event.base_asset_amount) self.assertEqual("BTC", order_completed_event.base_asset) - self.assertEqual("USD", order_completed_event.quote_asset) + self.assertEqual("USDT", order_completed_event.quote_asset) self.assertAlmostEqual(base_amount_traded, order_completed_event.base_asset_amount) self.assertAlmostEqual(quote_amount_traded, order_completed_event.quote_asset_amount) self.assertGreater(order_completed_event.fee_amount, Decimal(0)) @@ -206,7 +206,7 @@ def test_buy_and_sell(self): def test_limit_makers_unfilled(self): price = self.connector.get_price(self.trading_pair, True) * Decimal("0.8") price = self.connector.quantize_order_price(self.trading_pair, price) - amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0002")) self.ev_loop.run_until_complete(asyncio.sleep(1)) self.ev_loop.run_until_complete(self.connector._update_balances()) self.ev_loop.run_until_complete(asyncio.sleep(2)) @@ -231,7 +231,7 @@ def test_limit_makers_unfilled(self): price = self.connector.get_price(self.trading_pair, True) * Decimal("1.2") price = self.connector.quantize_order_price(self.trading_pair, price) - amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0002")) cl_order_id = self._place_order(False, amount, OrderType.LIMIT_MAKER, price, 2) order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCreatedEvent)) @@ -261,7 +261,7 @@ def test_cancel_all(self): ask_price = self.connector.get_price(self.trading_pair, False) bid_price = self.connector.quantize_order_price(self.trading_pair, bid_price * Decimal("0.9")) ask_price = self.connector.quantize_order_price(self.trading_pair, ask_price * Decimal("1.1")) - amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0002")) buy_id = self._place_order(True, amount, OrderType.LIMIT, bid_price, 1) sell_id = self._place_order(False, amount, OrderType.LIMIT, ask_price, 2) @@ -280,14 +280,14 @@ def test_order_quantized_values(self): # Make sure there's enough balance to make the limit orders. self.assertGreater(self.connector.get_balance("BTC"), Decimal("0.0005")) - self.assertGreater(self.connector.get_balance("USD"), Decimal("10")) + self.assertGreater(self.connector.get_balance("USDT"), Decimal("10")) # Intentionally set some prices with too many decimal places s.t. they # need to be quantized. Also, place them far away from the mid-price s.t. they won't # get filled during the test. bid_price = self.connector.quantize_order_price(self.trading_pair, mid_price * Decimal("0.9333192292111341")) ask_price = self.connector.quantize_order_price(self.trading_pair, mid_price * Decimal("1.1492431474884933")) - amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.000123456")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.000223456")) # Test bid order cl_order_id_1 = self._place_order(True, amount, OrderType.LIMIT, bid_price, 1) @@ -321,7 +321,7 @@ def test_orders_saving_and_restoration(self): price: Decimal = current_bid_price * Decimal("0.8") price = self.connector.quantize_order_price(self.trading_pair, price) - amount: Decimal = Decimal("0.0001") + amount: Decimal = Decimal("0.0002") amount = self.connector.quantize_order_amount(self.trading_pair, amount) cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1) @@ -402,7 +402,7 @@ def test_filled_orders_recorded(self): # Try to buy some token from the exchange, and watch for completion event. price = self.connector.get_price(self.trading_pair, True) * Decimal("1.05") price = self.connector.quantize_order_price(self.trading_pair, price) - amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0002")) order_id = self._place_order(True, amount, OrderType.LIMIT, price, 1) self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCompletedEvent)) @@ -414,7 +414,7 @@ def test_filled_orders_recorded(self): # Try to sell back the same amount to the exchange, and watch for completion event. price = self.connector.get_price(self.trading_pair, True) * Decimal("0.95") price = self.connector.quantize_order_price(self.trading_pair, price) - amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0002")) order_id = self._place_order(False, amount, OrderType.LIMIT, price, 2) self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCompletedEvent)) self.ev_loop.run_until_complete(asyncio.sleep(1)) diff --git a/test/connector/exchange/hitbtc/test_hitbtc_order_book_tracker.py b/test/connector/exchange/hitbtc/test_hitbtc_order_book_tracker.py index ae3778e7c9..bb929c3474 100755 --- a/test/connector/exchange/hitbtc/test_hitbtc_order_book_tracker.py +++ b/test/connector/exchange/hitbtc/test_hitbtc_order_book_tracker.py @@ -25,8 +25,8 @@ class HitbtcOrderBookTrackerUnitTest(unittest.TestCase): OrderBookEvent.TradeEvent ] trading_pairs: List[str] = [ - "BTC-USD", - "ETH-USD", + "BTC-USDT", + "ETH-USDT", ] @classmethod @@ -87,7 +87,7 @@ def test_tracker_integrity(self): # Wait 5 seconds to process some diffs. self.ev_loop.run_until_complete(asyncio.sleep(5.0)) order_books: Dict[str, OrderBook] = self.order_book_tracker.order_books - eth_usd: OrderBook = order_books["ETH-USD"] + eth_usd: OrderBook = order_books["ETH-USDT"] self.assertIsNot(eth_usd.last_diff_uid, 0) self.assertGreaterEqual(eth_usd.get_price_for_volume(True, 10).result_price, eth_usd.get_price(True)) @@ -96,8 +96,8 @@ def test_tracker_integrity(self): def test_api_get_last_traded_prices(self): prices = self.ev_loop.run_until_complete( - HitbtcAPIOrderBookDataSource.get_last_traded_prices(["BTC-USD", "LTC-BTC"])) + HitbtcAPIOrderBookDataSource.get_last_traded_prices(["BTC-USDT", "LTC-BTC"])) for key, value in prices.items(): print(f"{key} last_trade_price: {value}") - self.assertGreater(prices["BTC-USD"], 1000) + self.assertGreater(prices["BTC-USDT"], 1000) self.assertLess(prices["LTC-BTC"], 1) diff --git a/test/connector/exchange/hitbtc/test_hitbtc_user_stream_tracker.py b/test/connector/exchange/hitbtc/test_hitbtc_user_stream_tracker.py index 5c82f2372b..c53dcff7bc 100644 --- a/test/connector/exchange/hitbtc/test_hitbtc_user_stream_tracker.py +++ b/test/connector/exchange/hitbtc/test_hitbtc_user_stream_tracker.py @@ -24,7 +24,7 @@ class HitbtcUserStreamTrackerUnitTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() - cls.trading_pairs = ["BTC-USD"] + cls.trading_pairs = ["BTC-USDT"] cls.user_stream_tracker: HitbtcUserStreamTracker = HitbtcUserStreamTracker( hitbtc_auth=HitbtcAuth(cls.api_key, cls.api_secret), trading_pairs=cls.trading_pairs) diff --git a/test/test_rate_oracle.py b/test/test_rate_oracle.py index 9ecefae688..cbe1fe8d32 100644 --- a/test/test_rate_oracle.py +++ b/test/test_rate_oracle.py @@ -59,3 +59,20 @@ def test_find_rate(self): self.assertEqual(rate, Decimal("0.5")) rate = find_rate(prices, "HBOT-GBP") self.assertEqual(rate, Decimal("75")) + + def test_get_binance_prices(self): + asyncio.get_event_loop().run_until_complete(self._test_get_binance_prices()) + + async def _test_get_binance_prices(self): + com_prices = await RateOracle.get_binance_prices_by_domain(RateOracle.binance_price_url) + print(com_prices) + self.assertGreater(len(com_prices), 1) + us_prices = await RateOracle.get_binance_prices_by_domain(RateOracle.binance_us_price_url, "USD") + print(us_prices) + self.assertGreater(len(us_prices), 1) + quotes = {p.split("-")[1] for p in us_prices} + self.assertEqual(len(quotes), 1) + self.assertEqual(list(quotes)[0], "USD") + combined_prices = await RateOracle.get_binance_prices() + self.assertGreater(len(combined_prices), 1) + self.assertGreater(len(combined_prices), len(com_prices))