diff --git a/README.md b/README.md index 10417f0950..f668e3d96f 100644 --- a/README.md +++ b/README.md @@ -24,18 +24,19 @@ We created hummingbot to promote **decentralized market-making**: enabling membe | logo | id | name | ver | doc | status | |:---:|:---:|:---:|:---:|:---:|:---:| +| AscendEx | ascend_ex | [AscendEx](https://ascendex.com/en/global-digital-asset-platform) | 1 | [API](https://ascendex.github.io/ascendex-pro-api/#ascendex-pro-api-documentation) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Beaxy | beaxy | [Beaxy](https://beaxy.com/) | 2 | [API](https://beaxyapiv2trading.docs.apiary.io/) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | | Binance | binance | [Binance](https://www.binance.com/) | 3 | [API](https://github.com/binance/binance-spot-api-docs/blob/master/rest-api.md) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Binance US | binance_us | [Binance US](https://www.binance.com/) | 3 | [API](https://github.com/binance-us/binance-official-api-docs/blob/master/rest-api.md) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | | Binance Perpetual | binance_perpetual | [Binance Futures](https://www.binance.com/) | 1 | [API](https://binance-docs.github.io/apidocs/futures/en/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | |Bittrex Global| bittrex | [Bittrex Global](https://global.bittrex.com/) | 3 | [API](https://bittrex.github.io/api/v3) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | | Bitfinex | bitfinex | [Bitfinex](https://www.bitfinex.com/) | 2 | [API](https://docs.bitfinex.com/docs/introduction) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | -| BitMax | bitmax | [BitMax](https://bitmax.io/en/global-digital-asset-platform) | 1 | [API](https://bitmax-exchange.github.io/bitmax-pro-api/#bitmax-pro-api-documentation) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Blocktane | blocktane | [Blocktane](https://blocktane.io/) | 2 | [API](https://blocktane.io/api) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Coinbase Pro | coinbase_pro | [Coinbase Pro](https://pro.coinbase.com/) | * | [API](https://docs.pro.coinbase.com/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Crypto.com | crypto_com | [Crypto.com](https://crypto.com/exchange) | 2 | [API](https://exchange-docs.crypto.com/#introduction) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | | DyDx | dydx | [dy/dx](https://dydx.exchange/) | 1 | [API](https://docs.dydx.exchange/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Eterbase | eterbase | [Eterbase](https://www.eterbase.com/) | * | [API](https://developers.eterbase.exchange/?version=latest) |![RED](https://via.placeholder.com/15/f03c15/?text=+) | +| HitBTC | hitbtc | [HitBTC](https://hitbtc.com/) | 2 | [API](https://api.hitbtc.com/) | ![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | |Huobi Global| huobi | [Huobi Global](https://www.hbg.com) | 1 | [API](https://huobiapi.github.io/docs/spot/v1/en/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | KuCoin | kucoin | [KuCoin](https://www.kucoin.com/) | 1 | [API](https://docs.kucoin.com/#general) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Kraken | kraken | [Kraken](https://www.kraken.com/) | 1 | [API](https://www.kraken.com/features/api) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | diff --git a/assets/ascend_ex_logo.png b/assets/ascend_ex_logo.png new file mode 100644 index 0000000000..c30f757cf4 Binary files /dev/null and b/assets/ascend_ex_logo.png differ diff --git a/assets/bitmax_logo.png b/assets/bitmax_logo.png deleted file mode 100644 index 362daeca21..0000000000 Binary files a/assets/bitmax_logo.png and /dev/null differ diff --git a/assets/hitbtc_logo.png b/assets/hitbtc_logo.png new file mode 100644 index 0000000000..efdac10517 Binary files /dev/null and b/assets/hitbtc_logo.png differ diff --git a/bin/hummingbot.py b/bin/hummingbot.py index 07c0d7c886..2ae42f4a66 100755 --- a/bin/hummingbot.py +++ b/bin/hummingbot.py @@ -45,7 +45,7 @@ async def main(): # This init_logging() call is important, to skip over the missing config warnings. init_logging("hummingbot_logs.yml") - read_system_configs_from_yml() + await read_system_configs_from_yml() hb = HummingbotApplication.main_application() diff --git a/bin/hummingbot_quickstart.py b/bin/hummingbot_quickstart.py index 5f940c7758..91b75c25a4 100755 --- a/bin/hummingbot_quickstart.py +++ b/bin/hummingbot_quickstart.py @@ -78,14 +78,14 @@ async def quick_start(args): await Security.wait_til_decryption_done() await create_yml_files() init_logging("hummingbot_logs.yml") - read_system_configs_from_yml() + await read_system_configs_from_yml() hb = HummingbotApplication.main_application() # Todo: validate strategy and config_file_name before assinging if config_file_name is not None: hb.strategy_file_name = config_file_name - hb.strategy_name = update_strategy_config_map_from_file(os.path.join(CONF_FILE_PATH, config_file_name)) + hb.strategy_name = await update_strategy_config_map_from_file(os.path.join(CONF_FILE_PATH, config_file_name)) # To ensure quickstart runs with the default value of False for kill_switch_enabled if not present if not global_config_map.get("kill_switch_enabled"): diff --git a/conf/__init__.py b/conf/__init__.py index 65ba5a8347..7854c51a99 100644 --- a/conf/__init__.py +++ b/conf/__init__.py @@ -104,6 +104,10 @@ crypto_com_api_key = os.getenv("CRYPTO_COM_API_KEY") crypto_com_secret_key = os.getenv("CRYPTO_COM_SECRET_KEY") +# HitBTC Tests +hitbtc_api_key = os.getenv("HITBTC_API_KEY") +hitbtc_secret_key = os.getenv("HITBTC_SECRET_KEY") + # Wallet Tests test_erc20_token_address = os.getenv("TEST_ERC20_TOKEN_ADDRESS") web3_test_private_key_a = os.getenv("TEST_WALLET_PRIVATE_KEY_A") diff --git a/hummingbot/client/command/__init__.py b/hummingbot/client/command/__init__.py index 738e615143..a869e0a0fd 100644 --- a/hummingbot/client/command/__init__.py +++ b/hummingbot/client/command/__init__.py @@ -19,6 +19,7 @@ from .open_orders_command import OpenOrdersCommand from .trades_command import TradesCommand from .pnl_command import PnlCommand +from .rate_command import RateCommand __all__ = [ @@ -42,5 +43,6 @@ GenerateCertsCommand, OpenOrdersCommand, TradesCommand, - PnlCommand + PnlCommand, + RateCommand, ] diff --git a/hummingbot/client/command/balance_command.py b/hummingbot/client/command/balance_command.py index 386fa731bf..459ac95eab 100644 --- a/hummingbot/client/command/balance_command.py +++ b/hummingbot/client/command/balance_command.py @@ -10,7 +10,8 @@ ) from hummingbot.client.config.config_validators import validate_decimal, validate_exchange from hummingbot.market.celo.celo_cli import CeloCLI -from hummingbot.core.utils.market_price import usd_value +from hummingbot.client.performance import smart_round +from hummingbot.core.rate_oracle.rate_oracle import RateOracle import pandas as pd from decimal import Decimal from typing import TYPE_CHECKING, Dict, Optional, List @@ -76,6 +77,7 @@ def balance(self, save_to_yml(file_path, config_map) async def show_balances(self): + total_col_name = f'Total ({RateOracle.global_token_symbol})' self._notify("Updating balances, please wait...") all_ex_bals = await UserBalances.instance().all_balances_all_exchanges() all_ex_avai_bals = UserBalances.instance().all_avai_balances_all_exchanges() @@ -88,18 +90,17 @@ async def show_balances(self): for exchange, bals in all_ex_bals.items(): self._notify(f"\n{exchange}:") - # df = await self.exchange_balances_df(bals, all_ex_limits.get(exchange, {})) - df, allocated_total = await self.exchange_balances_usd_df(bals, all_ex_avai_bals.get(exchange, {})) + df, allocated_total = await self.exchange_balances_extra_df(bals, all_ex_avai_bals.get(exchange, {})) if df.empty: self._notify("You have no balance on this exchange.") else: lines = [" " + line for line in df.to_string(index=False).split("\n")] self._notify("\n".join(lines)) - self._notify(f"\n Total: $ {df['Total ($)'].sum():.0f} " - f"Allocated: {allocated_total / df['Total ($)'].sum():.2%}") - exchanges_total += df['Total ($)'].sum() + self._notify(f"\n Total: {RateOracle.global_token_symbol} {smart_round(df[total_col_name].sum())} " + f"Allocated: {allocated_total / df[total_col_name].sum():.2%}") + exchanges_total += df[total_col_name].sum() - self._notify(f"\n\nExchanges Total: $ {exchanges_total:.0f} ") + self._notify(f"\n\nExchanges Total: {RateOracle.global_token_symbol} {exchanges_total:.0f} ") celo_address = global_config_map["celo_address"].value if celo_address is not None: @@ -126,25 +127,10 @@ async def show_balances(self): self._notify("\nxdai:") self._notify("\n".join(lines)) - async def exchange_balances_df(self, # type: HummingbotApplication - exchange_balances: Dict[str, Decimal], - exchange_limits: Dict[str, str]): - rows = [] - for token, bal in exchange_balances.items(): - limit = Decimal(exchange_limits.get(token.upper(), 0)) if exchange_limits is not None else Decimal(0) - if bal == Decimal(0) and limit == Decimal(0): - continue - token = token.upper() - rows.append({"Asset": token.upper(), - "Amount": round(bal, 4), - "Limit": round(limit, 4) if limit > Decimal(0) else "-"}) - df = pd.DataFrame(data=rows, columns=["Asset", "Amount", "Limit"]) - df.sort_values(by=["Asset"], inplace=True) - return df - - async def exchange_balances_usd_df(self, # type: HummingbotApplication - ex_balances: Dict[str, Decimal], - ex_avai_balances: Dict[str, Decimal]): + async def exchange_balances_extra_df(self, # type: HummingbotApplication + ex_balances: Dict[str, Decimal], + ex_avai_balances: Dict[str, Decimal]): + total_col_name = f"Total ({RateOracle.global_token_symbol})" allocated_total = Decimal("0") rows = [] for token, bal in ex_balances.items(): @@ -152,15 +138,16 @@ async def exchange_balances_usd_df(self, # type: HummingbotApplication continue avai = Decimal(ex_avai_balances.get(token.upper(), 0)) if ex_avai_balances is not None else Decimal(0) allocated = f"{(bal - avai) / bal:.0%}" - usd = await usd_value(token, bal) - usd = 0 if usd is None else usd - allocated_total += await usd_value(token, (bal - avai)) + rate = await RateOracle.global_rate(token) + rate = Decimal("0") if rate is None else rate + global_value = rate * bal + allocated_total += rate * (bal - avai) rows.append({"Asset": token.upper(), "Total": round(bal, 4), - "Total ($)": round(usd), + total_col_name: smart_round(global_value), "Allocated": allocated, }) - df = pd.DataFrame(data=rows, columns=["Asset", "Total", "Total ($)", "Allocated"]) + df = pd.DataFrame(data=rows, columns=["Asset", "Total", total_col_name, "Allocated"]) df.sort_values(by=["Asset"], inplace=True) return df, allocated_total diff --git a/hummingbot/client/command/config_command.py b/hummingbot/client/command/config_command.py index 444609ce50..2b6a7b178a 100644 --- a/hummingbot/client/command/config_command.py +++ b/hummingbot/client/command/config_command.py @@ -36,6 +36,7 @@ no_restart_pmm_keys_in_percentage = ["bid_spread", "ask_spread", "order_level_spread", "inventory_target_base_pct"] no_restart_pmm_keys = ["order_amount", "order_levels", "filled_order_delay", "inventory_skew_enabled", "inventory_range_multiplier"] global_configs_to_display = ["0x_active_cancels", + "autofill_import", "kill_switch_enabled", "kill_switch_rate", "telegram_enabled", @@ -44,17 +45,15 @@ "send_error_logs", "script_enabled", "script_file_path", - "manual_gas_price", "ethereum_chain_name", - "ethgasstation_gas_enabled", - "ethgasstation_api_key", - "ethgasstation_gas_level", - "ethgasstation_refresh_time", "gateway_enabled", "gateway_cert_passphrase", "gateway_api_host", "gateway_api_port", - "balancer_max_swaps"] + "balancer_max_swaps", + "rate_oracle_source", + "global_token", + "global_token_symbol"] class ConfigCommand: @@ -199,7 +198,7 @@ async def asset_ratio_maintenance_prompt(self, # type: HummingbotApplication balances = await UserBalances.instance().balances(exchange, base, quote) if balances is None: return - base_ratio = UserBalances.base_amount_ratio(exchange, market, balances) + base_ratio = await UserBalances.base_amount_ratio(exchange, market, balances) if base_ratio is None: return base_ratio = round(base_ratio, 3) diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index 21881542b7..eb6d6e7487 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -100,11 +100,12 @@ async def prompt_a_config(self, # type: HummingbotApplication if input_value is None: if assign_default: self.app.set_text(parse_config_default_to_text(config)) - input_value = await self.app.prompt(prompt=config.prompt, is_password=config.is_secure) + prompt = await config.get_prompt() + input_value = await self.app.prompt(prompt=prompt, is_password=config.is_secure) if self.app.to_stop_config: return - err_msg = config.validate(input_value) + err_msg = await config.validate(input_value) if err_msg is not None: self._notify(err_msg) await self.prompt_a_config(config) diff --git a/hummingbot/client/command/import_command.py b/hummingbot/client/command/import_command.py index 8c02162393..5799f3a2f4 100644 --- a/hummingbot/client/command/import_command.py +++ b/hummingbot/client/command/import_command.py @@ -1,6 +1,7 @@ import os from hummingbot.core.utils.async_utils import safe_ensure_future +from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.config_helpers import ( update_strategy_config_map_from_file, short_strategy_name, @@ -34,7 +35,7 @@ async def import_config_file(self, # type: HummingbotApplication self.app.to_stop_config = False return strategy_path = os.path.join(CONF_FILE_PATH, file_name) - strategy = update_strategy_config_map_from_file(strategy_path) + strategy = await update_strategy_config_map_from_file(strategy_path) self.strategy_file_name = file_name self.strategy_name = strategy self._notify(f"Configuration from {self.strategy_file_name} file is imported.") @@ -43,6 +44,9 @@ async def import_config_file(self, # type: HummingbotApplication self.app.change_prompt(prompt=">>> ") if await self.status_check_all(): self._notify("\nEnter \"start\" to start market making.") + autofill_import = global_config_map.get("autofill_import").value + if autofill_import is not None: + self.app.set_text(autofill_import) async def prompt_a_file_name(self # type: HummingbotApplication ): diff --git a/hummingbot/client/command/open_orders_command.py b/hummingbot/client/command/open_orders_command.py index ea5a2dde2e..6e4fd36966 100644 --- a/hummingbot/client/command/open_orders_command.py +++ b/hummingbot/client/command/open_orders_command.py @@ -9,7 +9,8 @@ from datetime import timezone from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.core.data_type.common import OpenOrder -from hummingbot.core.utils.market_price import usd_value, get_binance_mid_price +from hummingbot.core.utils.market_price import get_binance_mid_price +from hummingbot.core.rate_oracle.rate_oracle import RateOracle s_float_0 = float(0) s_decimal_0 = Decimal("0") @@ -30,6 +31,7 @@ async def open_orders_report(self, # type: HummingbotApplication full_report: bool): exchange = "binance" connector = await self.get_binance_connector() + g_sym = RateOracle.global_token_symbol if connector is None: self._notify("This command supports only binance (for now), please first connect to binance.") return @@ -39,31 +41,31 @@ async def open_orders_report(self, # type: HummingbotApplication return orders = sorted(orders, key=lambda x: (x.trading_pair, x.is_buy)) data = [] - columns = ["Market", " Side", " Spread", " Size ($)", " Age"] + columns = ["Market", " Side", " Spread", f" Size ({g_sym})", " Age"] if full_report: columns.extend([" Allocation", " Per Total"]) cur_balances = await self.get_current_balances(exchange) total_value = 0 for o in orders: - total_value += await usd_value(o.trading_pair.split("-")[0], o.amount) + total_value += await RateOracle.global_value(o.trading_pair.split("-")[0], o.amount) for order in orders: base, quote = order.trading_pair.split("-") side = "buy" if order.is_buy else "sell" mid_price = await get_binance_mid_price(order.trading_pair) spread = abs(order.price - mid_price) / mid_price - size_usd = await usd_value(order.trading_pair.split("-")[0], order.amount) + size_global = await RateOracle.global_value(order.trading_pair.split("-")[0], order.amount) age = pd.Timestamp((datetime.utcnow().replace(tzinfo=timezone.utc).timestamp() * 1e3 - order.time) / 1e3, unit='s').strftime('%H:%M:%S') - data_row = [order.trading_pair, side, f"{spread:.2%}", round(size_usd), age] + data_row = [order.trading_pair, side, f"{spread:.2%}", round(size_global), age] if full_report: token = quote if order.is_buy else base token_value = order.amount * order.price if order.is_buy else order.amount per_bal = token_value / cur_balances[token] token_txt = f"({token})" - data_row.extend([f"{per_bal:.0%} {token_txt:>6}", f"{size_usd / total_value:.0%}"]) + data_row.extend([f"{per_bal:.0%} {token_txt:>6}", f"{size_global / total_value:.0%}"]) data.append(data_row) lines = [] orders_df: pd.DataFrame = pd.DataFrame(data=data, columns=columns) lines.extend([" " + line for line in orders_df.to_string(index=False).split("\n")]) self._notify("\n" + "\n".join(lines)) - self._notify(f"\n Total: $ {total_value:.0f}") + self._notify(f"\n Total: {g_sym} {total_value:.0f}") diff --git a/hummingbot/client/command/pnl_command.py b/hummingbot/client/command/pnl_command.py index f02c07358c..75a1ebf720 100644 --- a/hummingbot/client/command/pnl_command.py +++ b/hummingbot/client/command/pnl_command.py @@ -9,12 +9,12 @@ from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.client.config.security import Security from hummingbot.user.user_balances import UserBalances -from hummingbot.core.utils.market_price import usd_value from hummingbot.core.data_type.trade import Trade from hummingbot.core.data_type.common import OpenOrder from hummingbot.client.performance import calculate_performance_metrics from hummingbot.client.command.history_command import get_timestamp from hummingbot.client.config.global_config_map import global_config_map +from hummingbot.core.rate_oracle.rate_oracle import RateOracle s_float_0 = float(0) s_decimal_0 = Decimal("0") @@ -73,16 +73,17 @@ async def pnl_report(self, # type: HummingbotApplication markets = set(global_config_map["binance_markets"].value.split(",")) markets = sorted(markets) data = [] - columns = ["Market", " Traded ($)", " Fee ($)", " PnL ($)", " Return %"] + g_sym = RateOracle.global_token_symbol + columns = ["Market", f" Traded ({g_sym})", f" Fee ({g_sym})", f" PnL ({g_sym})", " Return %"] for market in markets: base, quote = market.split("-") trades: List[Trade] = await connector.get_my_trades(market, days) if not trades: continue perf = await calculate_performance_metrics(exchange, market, trades, cur_balances) - volume = await usd_value(quote, abs(perf.b_vol_quote) + abs(perf.s_vol_quote)) - fee = await usd_value(quote, perf.fee_in_quote) - pnl = await usd_value(quote, perf.total_pnl) + volume = await RateOracle.global_value(quote, abs(perf.b_vol_quote) + abs(perf.s_vol_quote)) + fee = await RateOracle.global_value(quote, perf.fee_in_quote) + pnl = await RateOracle.global_value(quote, perf.total_pnl) data.append([market, round(volume, 2), round(fee, 2), round(pnl, 2), f"{perf.return_pct:.2%}"]) if not data: self._notify(f"No trades during the last {days} day(s).") @@ -91,4 +92,5 @@ async def pnl_report(self, # type: HummingbotApplication df: pd.DataFrame = pd.DataFrame(data=data, columns=columns) lines.extend([" " + line for line in df.to_string(index=False).split("\n")]) self._notify("\n" + "\n".join(lines)) - self._notify(f"\n Total PnL: $ {df[' PnL ($)'].sum():.2f} Total fee: $ {df[' Fee ($)'].sum():.2f}") + self._notify(f"\n Total PnL: {g_sym} {df[f' PnL ({g_sym})'].sum():.2f} " + f"Total fee: {g_sym} {df[f' Fee ({g_sym})'].sum():.2f}") diff --git a/hummingbot/client/command/rate_command.py b/hummingbot/client/command/rate_command.py new file mode 100644 index 0000000000..a23135c0dd --- /dev/null +++ b/hummingbot/client/command/rate_command.py @@ -0,0 +1,50 @@ +from decimal import Decimal +import threading +from typing import ( + TYPE_CHECKING, +) +from hummingbot.core.utils.async_utils import safe_ensure_future +from hummingbot.core.rate_oracle.rate_oracle import RateOracle + +s_float_0 = float(0) +s_decimal_0 = Decimal("0") + +if TYPE_CHECKING: + from hummingbot.client.hummingbot_application import HummingbotApplication + + +class RateCommand: + def rate(self, # type: HummingbotApplication + pair: str, + token: str + ): + if threading.current_thread() != threading.main_thread(): + self.ev_loop.call_soon_threadsafe(self.trades) + return + if pair: + safe_ensure_future(self.show_rate(pair)) + elif token: + safe_ensure_future(self.show_token_value(token)) + + async def show_rate(self, # type: HummingbotApplication + pair: str, + ): + pair = pair.upper() + self._notify(f"Source: {RateOracle.source.name}") + rate = await RateOracle.rate_async(pair) + if rate is None: + self._notify("Rate is not available.") + return + base, quote = pair.split("-") + self._notify(f"1 {base} = {rate} {quote}") + + async def show_token_value(self, # type: HummingbotApplication + token: str + ): + token = token.upper() + self._notify(f"Source: {RateOracle.source.name}") + rate = await RateOracle.global_rate(token) + if rate is None: + self._notify("Rate is not available.") + return + self._notify(f"1 {token} = {RateOracle.global_token_symbol} {rate} {RateOracle.global_token}") diff --git a/hummingbot/client/command/start_command.py b/hummingbot/client/command/start_command.py index e8c2e589a3..3c36475ee1 100644 --- a/hummingbot/client/command/start_command.py +++ b/hummingbot/client/command/start_command.py @@ -20,14 +20,12 @@ from hummingbot.client.settings import ( STRATEGIES, SCRIPTS_PATH, - ethereum_gas_station_required, required_exchanges, ) from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.core.utils.kill_switch import KillSwitch from typing import TYPE_CHECKING from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.core.utils.eth_gas_station_lookup import EthGasStationLookup from hummingbot.script.script_iterator import ScriptIterator from hummingbot.connector.connector_status import get_connector_status, warning_messages if TYPE_CHECKING: @@ -142,9 +140,6 @@ async def start_market_making(self, # type: HummingbotApplication self.clock.add_iterator(self._script_iterator) self._notify(f"Script ({script_file}) started.") - if global_config_map["ethgasstation_gas_enabled"].value and ethereum_gas_station_required(): - EthGasStationLookup.get_instance().start() - self.strategy_task: asyncio.Task = safe_ensure_future(self._run_clock(), loop=self.ev_loop) self._notify(f"\n'{strategy_name}' strategy started.\n" f"Run `status` command to query the progress.") diff --git a/hummingbot/client/command/status_command.py b/hummingbot/client/command/status_command.py index 3cf92785ad..1fb5e69344 100644 --- a/hummingbot/client/command/status_command.py +++ b/hummingbot/client/command/status_command.py @@ -18,7 +18,7 @@ ) from hummingbot.client.config.security import Security from hummingbot.user.user_balances import UserBalances -from hummingbot.client.settings import required_exchanges, ethereum_wallet_required, ethereum_gas_station_required +from hummingbot.client.settings import required_exchanges, ethereum_wallet_required from hummingbot.core.utils.async_utils import safe_ensure_future from typing import TYPE_CHECKING @@ -186,10 +186,6 @@ async def status_check_all(self, # type: HummingbotApplication else: self._notify(" - ETH wallet check: ETH wallet is not connected.") - if ethereum_gas_station_required() and not global_config_map["ethgasstation_gas_enabled"].value: - self._notify(f' - ETH gas station check: Manual gas price is fixed at ' - f'{global_config_map["manual_gas_price"].value}.') - loading_markets: List[ConnectorBase] = [] for market in self.markets.values(): if not market.ready: diff --git a/hummingbot/client/command/stop_command.py b/hummingbot/client/command/stop_command.py index 3848aadea4..6eac19b03e 100644 --- a/hummingbot/client/command/stop_command.py +++ b/hummingbot/client/command/stop_command.py @@ -3,7 +3,6 @@ import threading from typing import TYPE_CHECKING from hummingbot.core.utils.async_utils import safe_ensure_future -from hummingbot.core.utils.eth_gas_station_lookup import EthGasStationLookup if TYPE_CHECKING: from hummingbot.client.hummingbot_application import HummingbotApplication @@ -45,9 +44,6 @@ async def stop_loop(self, # type: HummingbotApplication if self.strategy_task is not None and not self.strategy_task.cancelled(): self.strategy_task.cancel() - if EthGasStationLookup.get_instance().started: - EthGasStationLookup.get_instance().stop() - if self.markets_recorder is not None: self.markets_recorder.stop() diff --git a/hummingbot/client/command/trades_command.py b/hummingbot/client/command/trades_command.py index d319aee011..bc1b502a1e 100644 --- a/hummingbot/client/command/trades_command.py +++ b/hummingbot/client/command/trades_command.py @@ -7,12 +7,12 @@ ) from datetime import datetime from hummingbot.core.utils.async_utils import safe_ensure_future -from hummingbot.core.utils.market_price import usd_value from hummingbot.core.data_type.trade import Trade, TradeType from hummingbot.core.data_type.common import OpenOrder from hummingbot.client.performance import smart_round from hummingbot.client.command.history_command import get_timestamp from hummingbot.client.config.global_config_map import global_config_map +from hummingbot.core.rate_oracle.rate_oracle import RateOracle s_float_0 = float(0) s_decimal_0 = Decimal("0") @@ -58,11 +58,13 @@ async def market_trades_report(self, # type: HummingbotApplication days: float, market: str): trades: List[Trade] = await connector.get_my_trades(market, days) + g_sym = RateOracle.global_token_symbol if not trades: self._notify(f"There is no trade on {market}.") return data = [] - columns = ["Time", " Side", " Price", "Amount", " Amount ($)"] + amount_g_col_name = f" Amount ({g_sym})" + columns = ["Time", " Side", " Price", "Amount", amount_g_col_name] trades = sorted(trades, key=lambda x: (x.trading_pair, x.timestamp)) fees = {} # a dict of token and total fee amount fee_usd = 0 @@ -70,14 +72,14 @@ async def market_trades_report(self, # type: HummingbotApplication for trade in trades: time = f"{datetime.fromtimestamp(trade.timestamp / 1e3).strftime('%Y-%m-%d %H:%M:%S')} " side = "buy" if trade.side is TradeType.BUY else "sell" - usd = await usd_value(trade.trading_pair.split("-")[0], trade.amount) + usd = await RateOracle.global_value(trade.trading_pair.split("-")[0], trade.amount) data.append([time, side, smart_round(trade.price), smart_round(trade.amount), round(usd)]) for fee in trade.trade_fee.flat_fees: if fee[0] not in fees: fees[fee[0]] = fee[1] else: fees[fee[0]] += fee[1] - fee_usd += await usd_value(fee[0], fee[1]) + fee_usd += await RateOracle.global_value(fee[0], fee[1]) lines = [] df: pd.DataFrame = pd.DataFrame(data=data, columns=columns) @@ -85,4 +87,5 @@ async def market_trades_report(self, # type: HummingbotApplication lines.extend([" " + line for line in df.to_string(index=False).split("\n")]) self._notify("\n" + "\n".join(lines)) fee_text = ",".join(k + ": " + f"{v:.4f}" for k, v in fees.items()) - self._notify(f"\n Total traded: $ {df[' Amount ($)'].sum():.0f} Fees: {fee_text} ($ {fee_usd:.2f})") + self._notify(f"\n Total traded: {g_sym} {df[amount_g_col_name].sum():.0f} " + f"Fees: {fee_text} ({g_sym} {fee_usd:.2f})") diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index 3e4877a603..8bc0b2ebcc 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -1,5 +1,6 @@ import logging from decimal import Decimal +from functools import lru_cache import ruamel.yaml from os import ( unlink @@ -35,7 +36,7 @@ CONNECTOR_SETTINGS ) from hummingbot.client.config.security import Security -from hummingbot.core.utils.market_price import get_mid_price +from hummingbot.core.utils.market_price import get_last_price from hummingbot import get_strategy_list from eth_account import Account @@ -164,6 +165,7 @@ def get_eth_wallet_private_key() -> Optional[str]: return account.privateKey.hex() +@lru_cache(None) def get_erc20_token_addresses() -> Dict[str, List]: token_list_url = global_config_map.get("ethereum_token_list_url").value address_file_path = TOKEN_ADDRESSES_FILE_PATH @@ -261,15 +263,15 @@ def validate_strategy_file(file_path: str) -> Optional[str]: return None -def update_strategy_config_map_from_file(yml_path: str) -> str: +async def update_strategy_config_map_from_file(yml_path: str) -> str: strategy = strategy_name_from_file(yml_path) config_map = get_strategy_config_map(strategy) template_path = get_strategy_template_path(strategy) - load_yml_into_cm(yml_path, template_path, config_map) + await load_yml_into_cm(yml_path, template_path, config_map) return strategy -def load_yml_into_cm(yml_path: str, template_file_path: str, cm: Dict[str, ConfigVar]): +async def load_yml_into_cm(yml_path: str, template_file_path: str, cm: Dict[str, ConfigVar]): try: with open(yml_path) as stream: data = yaml_parser.load(stream) or {} @@ -301,7 +303,7 @@ def load_yml_into_cm(yml_path: str, template_file_path: str, cm: Dict[str, Confi # Todo: the proper process should be first validate the value then assign it cvar.value = parse_cvar_value(cvar, val_in_file) if cvar.value is not None: - err_msg = cvar.validate(str(cvar.value)) + err_msg = await cvar.validate(str(cvar.value)) if err_msg is not None: # Instead of raising an exception, simply skip over this variable and wait till the user is prompted logging.getLogger().error("Invalid value %s for config variable %s" % (val_in_file, cvar.key)) @@ -320,14 +322,14 @@ def load_yml_into_cm(yml_path: str, template_file_path: str, cm: Dict[str, Confi exc_info=True) -def read_system_configs_from_yml(): +async def read_system_configs_from_yml(): """ Read global config and selected strategy yml files and save the values to corresponding config map If a yml file is outdated, it gets reformatted with the new template """ - load_yml_into_cm(GLOBAL_CONFIG_PATH, join(TEMPLATE_PATH, "conf_global_TEMPLATE.yml"), global_config_map) - load_yml_into_cm(TRADE_FEES_CONFIG_PATH, join(TEMPLATE_PATH, "conf_fee_overrides_TEMPLATE.yml"), - fee_overrides_config_map) + await load_yml_into_cm(GLOBAL_CONFIG_PATH, join(TEMPLATE_PATH, "conf_global_TEMPLATE.yml"), global_config_map) + await load_yml_into_cm(TRADE_FEES_CONFIG_PATH, join(TEMPLATE_PATH, "conf_fee_overrides_TEMPLATE.yml"), + fee_overrides_config_map) # In case config maps get updated (due to default values) save_system_configs_to_yml() @@ -403,12 +405,12 @@ def default_min_quote(quote_asset: str) -> (str, Decimal): return result_quote, result_amount -def minimum_order_amount(exchange: str, trading_pair: str) -> Decimal: +async def minimum_order_amount(exchange: str, trading_pair: str) -> Decimal: base_asset, quote_asset = trading_pair.split("-") default_quote_asset, default_amount = default_min_quote(quote_asset) quote_amount = Decimal("0") if default_quote_asset == quote_asset: - mid_price = get_mid_price(exchange, trading_pair) + mid_price = await get_last_price(exchange, trading_pair) if mid_price is not None: quote_amount = default_amount / mid_price return round(quote_amount, 4) diff --git a/hummingbot/client/config/config_var.py b/hummingbot/client/config/config_var.py index b901fbdaff..85811bf2fb 100644 --- a/hummingbot/client/config/config_var.py +++ b/hummingbot/client/config/config_var.py @@ -1,17 +1,19 @@ from typing import ( Optional, - Callable, + Callable ) +import inspect RequiredIf = Callable[[str], Optional[bool]] Validator = Callable[[str], Optional[str]] +Prompt = Callable[[str], Optional[str]] OnValidated = Callable class ConfigVar: def __init__(self, key: str, - prompt: Optional[any], + prompt: Prompt, is_secure: bool = False, default: any = None, type_str: str = "str", @@ -23,7 +25,7 @@ def __init__(self, prompt_on_new: bool = False, # Whether this is a config var used in connect command is_connect_key: bool = False): - self._prompt = prompt + self.prompt = prompt self.key = key self.value = None self.is_secure = is_secure @@ -35,24 +37,32 @@ def __init__(self, self.prompt_on_new = prompt_on_new self.is_connect_key = is_connect_key - @property - def prompt(self): - if callable(self._prompt): - return self._prompt() + async def get_prompt(self): + if inspect.iscoroutinefunction(self.prompt): + return await self.prompt() + elif inspect.isfunction(self.prompt): + return self.prompt() else: - return self._prompt + return self.prompt @property def required(self) -> bool: assert callable(self._required_if) return self._required_if() - def validate(self, value: str) -> Optional[str]: + async def validate(self, value: str) -> Optional[str]: assert callable(self._validator) assert callable(self._on_validated) if self.required and (value is None or value == ""): return "Value is required." - err_msg = self._validator(value) + err_msg = None + if inspect.iscoroutinefunction(self._validator): + err_msg = await self._validator(value) + elif inspect.isfunction(self._validator): + err_msg = self._validator(value) if err_msg is None and self._validator is not None: - self._on_validated(value) + if inspect.iscoroutinefunction(self._on_validated): + await self._on_validated(value) + elif inspect.isfunction(self._on_validated): + self._on_validated(value) return err_msg diff --git a/hummingbot/client/config/global_config_map.py b/hummingbot/client/config/global_config_map.py index e6c7eaa141..57f37bbdb6 100644 --- a/hummingbot/client/config/global_config_map.py +++ b/hummingbot/client/config/global_config_map.py @@ -10,6 +10,7 @@ validate_int, validate_decimal ) +from hummingbot.core.rate_oracle.rate_oracle import RateOracleSource, RateOracle def generate_client_id() -> str: @@ -45,6 +46,23 @@ def connector_keys(): return all_keys +def validate_rate_oracle_source(value: str) -> Optional[str]: + if value not in (r.name for r in RateOracleSource): + return f"Invalid source, please choose value from {','.join(r.name for r in RateOracleSource)}" + + +def rate_oracle_source_on_validated(value: str): + RateOracle.source = RateOracleSource[value] + + +def global_token_on_validated(value: str): + RateOracle.global_token = value.upper() + + +def global_token_symbol_on_validated(value: str): + RateOracle.global_token_symbol = value + + # Main global config store key_config_map = connector_keys() @@ -178,6 +196,14 @@ def connector_keys(): default=-100, validator=lambda v: validate_decimal(v, Decimal(-100), Decimal(100)), required_if=lambda: global_config_map["kill_switch_enabled"].value), + "autofill_import": + ConfigVar(key="autofill_import", + prompt="What to auto-fill in the prompt after each import command? (start/config) >>> ", + type_str="str", + default=None, + validator=lambda s: None if s in {"start", + "config"} else "Invalid auto-fill prompt.", + required_if=lambda: False), "telegram_enabled": ConfigVar(key="telegram_enabled", prompt="Would you like to enable telegram? >>> ", @@ -272,32 +298,6 @@ def connector_keys(): type_str="decimal", validator=lambda v: validate_decimal(v, Decimal(0), inclusive=False), default=50), - "ethgasstation_gas_enabled": - ConfigVar(key="ethgasstation_gas_enabled", - prompt="Do you want to enable Ethereum gas station price lookup? >>> ", - required_if=lambda: False, - type_str="bool", - validator=validate_bool, - default=False), - "ethgasstation_api_key": - ConfigVar(key="ethgasstation_api_key", - prompt="Enter API key for defipulse.com gas station API >>> ", - required_if=lambda: global_config_map["ethgasstation_gas_enabled"].value, - type_str="str"), - "ethgasstation_gas_level": - ConfigVar(key="ethgasstation_gas_level", - prompt="Enter gas level you want to use for Ethereum transactions (fast, fastest, safeLow, average) " - ">>> ", - required_if=lambda: global_config_map["ethgasstation_gas_enabled"].value, - type_str="str", - validator=lambda s: None if s in {"fast", "fastest", "safeLow", "average"} - else "Invalid gas level."), - "ethgasstation_refresh_time": - ConfigVar(key="ethgasstation_refresh_time", - prompt="Enter refresh time for Ethereum gas price lookup (in seconds) >>> ", - required_if=lambda: global_config_map["ethgasstation_gas_enabled"].value, - type_str="int", - default=120), "gateway_api_host": ConfigVar(key="gateway_api_host", prompt=None, @@ -332,6 +332,29 @@ def connector_keys(): required_if=lambda: False, default="HARD-USDT,HARD-BTC,XEM-ETH,XEM-BTC,ALGO-USDT,ALGO-BTC,COTI-BNB,COTI-USDT,COTI-BTC,MFT-BNB," "MFT-ETH,MFT-USDT,RLC-ETH,RLC-BTC,RLC-USDT"), + "rate_oracle_source": + ConfigVar(key="rate_oracle_source", + prompt=f"What source do you want rate oracle to pull data from? " + f"({','.join(r.name for r in RateOracleSource)}) >>> ", + type_str="str", + required_if=lambda: False, + validator=validate_rate_oracle_source, + on_validated=rate_oracle_source_on_validated, + default=RateOracleSource.binance.name), + "global_token": + ConfigVar(key="global_token", + prompt="What is your default display token? (e.g. USDT,USD,EUR) >>> ", + type_str="str", + required_if=lambda: False, + on_validated=global_token_on_validated, + default="USDT"), + "global_token_symbol": + ConfigVar(key="global_token_symbol", + prompt="What is your default display token symbol? (e.g. $,€) >>> ", + type_str="str", + required_if=lambda: False, + on_validated=global_token_symbol_on_validated, + default="$"), } global_config_map = {**key_config_map, **main_config_map} diff --git a/hummingbot/client/hummingbot_application.py b/hummingbot/client/hummingbot_application.py index 3566917232..f777132f21 100644 --- a/hummingbot/client/hummingbot_application.py +++ b/hummingbot/client/hummingbot_application.py @@ -21,7 +21,6 @@ from hummingbot.client.errors import InvalidCommandError, ArgumentParserError from hummingbot.client.config.global_config_map import global_config_map, using_wallet from hummingbot.client.config.config_helpers import ( - get_erc20_token_addresses, get_strategy_config_map, get_connector_class, get_eth_wallet_private_key, @@ -29,6 +28,7 @@ from hummingbot.strategy.strategy_base import StrategyBase from hummingbot.strategy.cross_exchange_market_making import CrossExchangeMarketPair from hummingbot.core.utils.kill_switch import KillSwitch +from hummingbot.core.utils.trading_pair_fetcher import TradingPairFetcher from hummingbot.data_feed.data_feed_base import DataFeedBase from hummingbot.notifier.notifier_base import NotifierBase from hummingbot.notifier.telegram_notifier import TelegramNotifier @@ -36,7 +36,6 @@ from hummingbot.connector.markets_recorder import MarketsRecorder from hummingbot.client.config.security import Security from hummingbot.connector.exchange_base import ExchangeBase -from hummingbot.core.utils.trading_pair_fetcher import TradingPairFetcher from hummingbot.client.settings import CONNECTOR_SETTINGS, ConnectorType s_logger = None @@ -62,6 +61,8 @@ def main_application(cls) -> "HummingbotApplication": return cls._main_app def __init__(self): + # This is to start fetching trading pairs for auto-complete + TradingPairFetcher.get_instance() self.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() self.parser: ThrowingArgumentParser = load_parser(self) self.app = HummingbotCLI( @@ -96,9 +97,6 @@ def __init__(self): self.trade_fill_db: Optional[SQLConnectionManager] = None self.markets_recorder: Optional[MarketsRecorder] = None self._script_iterator = None - # This is to start fetching trading pairs for auto-complete - TradingPairFetcher.get_instance() - self._binance_connector = None @property @@ -195,10 +193,14 @@ def _initialize_market_assets(market_name: str, trading_pairs: List[str]) -> Lis return market_trading_pairs def _initialize_wallet(self, token_trading_pairs: List[str]): + # Todo: This function should be removed as it's currently not used by current working connectors + if not using_wallet(): return - if not self.token_list: - self.token_list = get_erc20_token_addresses() + # Commented this out for now since get_erc20_token_addresses uses blocking call + + # if not self.token_list: + # self.token_list = get_erc20_token_addresses() ethereum_wallet = global_config_map.get("ethereum_wallet").value private_key = Security._private_keys[ethereum_wallet] diff --git a/hummingbot/client/settings.py b/hummingbot/client/settings.py index ae6392fde7..c832bcc7e4 100644 --- a/hummingbot/client/settings.py +++ b/hummingbot/client/settings.py @@ -95,7 +95,7 @@ def base_name(self) -> str: def _create_connector_settings() -> Dict[str, ConnectorSetting]: - connector_exceptions = ["paper_trade"] + connector_exceptions = ["paper_trade", "eterbase"] connector_settings = {} package_dir = Path(__file__).resolve().parent.parent.parent type_dirs = [f for f in scandir(f'{str(package_dir)}/hummingbot/connector') if f.is_dir()] diff --git a/hummingbot/client/ui/completer.py b/hummingbot/client/ui/completer.py index ef85af2ccb..ba850a4bf1 100644 --- a/hummingbot/client/ui/completer.py +++ b/hummingbot/client/ui/completer.py @@ -21,6 +21,7 @@ from hummingbot.core.utils.wallet_setup import list_wallets from hummingbot.core.utils.trading_pair_fetcher import TradingPairFetcher from hummingbot.client.command.connect_command import OPTIONS as CONNECT_OPTIONS +from hummingbot.core.rate_oracle.rate_oracle import RateOracleSource def file_name_list(path, file_extension): @@ -39,9 +40,9 @@ def __init__(self, hummingbot_application): self.hummingbot_application = hummingbot_application self._path_completer = WordCompleter(file_name_list(CONF_FILE_PATH, "yml")) self._command_completer = WordCompleter(self.parser.commands, ignore_case=True) - self._exchange_completer = WordCompleter(CONNECTOR_SETTINGS.keys(), ignore_case=True) - self._spot_completer = WordCompleter(EXCHANGES.union(SPOT_PROTOCOL_CONNECTOR), ignore_case=True) - self._spot_exchange_completer = WordCompleter(EXCHANGES, ignore_case=True) + self._exchange_completer = WordCompleter(sorted(CONNECTOR_SETTINGS.keys()), ignore_case=True) + self._spot_completer = WordCompleter(sorted(EXCHANGES.union(SPOT_PROTOCOL_CONNECTOR)), ignore_case=True) + self._spot_exchange_completer = WordCompleter(sorted(EXCHANGES), ignore_case=True) self._derivative_completer = WordCompleter(DERIVATIVES, ignore_case=True) self._derivative_exchange_completer = WordCompleter(DERIVATIVES.difference(DERIVATIVE_PROTOCOL_CONNECTOR), ignore_case=True) self._connect_option_completer = WordCompleter(CONNECT_OPTIONS, ignore_case=True) @@ -50,6 +51,7 @@ def __init__(self, hummingbot_application): self._history_completer = WordCompleter(["--days", "--verbose", "--precision"], ignore_case=True) self._strategy_completer = WordCompleter(STRATEGIES, ignore_case=True) self._py_file_completer = WordCompleter(file_name_list(SCRIPTS_PATH, "py")) + self._rate_oracle_completer = WordCompleter([r.name for r in RateOracleSource], ignore_case=True) @property def prompt_text(self) -> str: @@ -156,6 +158,9 @@ def _complete_balance_limit_exchanges(self, document: Document): command_args = text_before_cursor.split(" ") return len(command_args) == 3 and command_args[0] == "balance" and command_args[1] == "limit" + def _complete_rate_oracle_source(self, document: Document): + return all(x in self.prompt_text for x in ("source", "rate oracle")) + def get_completions(self, document: Document, complete_event: CompleteEvent): """ Get completions for the current scope. This is the defining function for the completer @@ -234,6 +239,10 @@ def get_completions(self, document: Document, complete_event: CompleteEvent): for c in self._option_completer.get_completions(document, complete_event): yield c + elif self._complete_rate_oracle_source(document): + for c in self._rate_oracle_completer.get_completions(document, complete_event): + yield c + else: text_before_cursor: str = document.text_before_cursor try: diff --git a/hummingbot/client/ui/parser.py b/hummingbot/client/ui/parser.py index 06fb6a20da..f1fff6fa83 100644 --- a/hummingbot/client/ui/parser.py +++ b/hummingbot/client/ui/parser.py @@ -139,4 +139,11 @@ def load_parser(hummingbot) -> ThrowingArgumentParser: ticker_parser.add_argument("--market", type=str, dest="market", help="The market (trading pair) of the order book") ticker_parser.set_defaults(func=hummingbot.ticker) + rate_parser = subparsers.add_parser('rate', help="Show rate of a given trading pair") + rate_parser.add_argument("-p", "--pair", default=None, + dest="pair", help="The market trading pair you want to see rate.") + rate_parser.add_argument("-t", "--token", default=None, + dest="token", help="The token you want to see its value.") + rate_parser.set_defaults(func=hummingbot.rate) + return parser diff --git a/hummingbot/connector/connector/balancer/balancer_connector.py b/hummingbot/connector/connector/balancer/balancer_connector.py index e1bc1e131d..179c6b2e8f 100644 --- a/hummingbot/connector/connector/balancer/balancer_connector.py +++ b/hummingbot/connector/connector/balancer/balancer_connector.py @@ -7,7 +7,6 @@ import time import ssl import copy -import itertools as it from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL from hummingbot.core.utils import async_ttl_cache from hummingbot.core.network_iterator import NetworkStatus @@ -31,9 +30,9 @@ from hummingbot.connector.connector_base import ConnectorBase from hummingbot.connector.connector.balancer.balancer_in_flight_order import BalancerInFlightOrder from hummingbot.client.settings import GATEAWAY_CA_CERT_PATH, GATEAWAY_CLIENT_CERT_PATH, GATEAWAY_CLIENT_KEY_PATH -from hummingbot.core.utils.eth_gas_station_lookup import get_gas_price from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.client.config.config_helpers import get_erc20_token_addresses +from hummingbot.core.utils.ethereum import check_transaction_exceptions, fetch_trading_pairs +from hummingbot.client.config.fee_overrides_config_map import fee_overrides_config_map s_logger = None s_decimal_0 = Decimal("0") @@ -71,12 +70,9 @@ def __init__(self, """ super().__init__() self._trading_pairs = trading_pairs - tokens = set() + self._tokens = set() for trading_pair in trading_pairs: - tokens.update(set(trading_pair.split("-"))) - self._erc_20_token_list = self.token_list() - self._token_addresses = {t: l[0] for t, l in self._erc_20_token_list.items() if t in tokens} - self._token_decimals = {t: l[1] for t, l in self._erc_20_token_list.items() if t in tokens} + self._tokens.update(set(trading_pair.split("-"))) self._wallet_private_key = wallet_private_key self._ethereum_rpc_url = ethereum_rpc_url self._trading_required = trading_required @@ -84,10 +80,13 @@ def __init__(self, self._shared_client = None self._last_poll_timestamp = 0.0 self._last_balance_poll_timestamp = time.time() + self._last_est_gas_cost_reported = 0 self._in_flight_orders = {} self._allowances = {} self._status_polling_task = None self._auto_approve_task = None + self._initiate_pool_task = None + self._initiate_pool_status = None self._real_time_balance_update = False self._max_swaps = global_config_map['balancer_max_swaps'].value self._poll_notifier = None @@ -96,17 +95,9 @@ def __init__(self, def name(self): return "balancer" - @staticmethod - def token_list(): - return get_erc20_token_addresses() - @staticmethod async def fetch_trading_pairs() -> List[str]: - token_list = BalancerConnector.token_list() - trading_pairs = [] - for base, quote in it.permutations(token_list.keys(), 2): - trading_pairs.append(f"{base}-{quote}") - return trading_pairs + return await fetch_trading_pairs() @property def limit_orders(self) -> List[LimitOrder]: @@ -115,6 +106,26 @@ def limit_orders(self) -> List[LimitOrder]: for in_flight_order in self._in_flight_orders.values() ] + async def initiate_pool(self) -> str: + """ + Initiate connector and cache pools + """ + try: + self.logger().info(f"Initializing Balancer connector and caching pools for {self._trading_pairs}.") + resp = await self._api_request("get", "eth/balancer/start", + {"pairs": json.dumps(self._trading_pairs)}) + status = bool(str(resp["success"])) + if bool(str(resp["success"])): + self._initiate_pool_status = status + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().network( + f"Error initializing {self._trading_pairs} swap pools", + exc_info=True, + app_warning_msg=str(e) + ) + async def auto_approve(self): """ Automatically approves Balancer contract as a spender for token in trading pairs. @@ -138,9 +149,7 @@ async def approve_balancer_spender(self, token_symbol: str) -> Decimal: """ resp = await self._api_request("post", "eth/approve", - {"tokenAddress": self._token_addresses[token_symbol], - "gasPrice": str(get_gas_price()), - "decimals": self._token_decimals[token_symbol], # if not supplied, gateway would treat it eth-like with 18 decimals + {"token": token_symbol, "connector": self.name}) amount_approved = Decimal(str(resp["amount"])) if amount_approved > 0: @@ -155,12 +164,11 @@ async def get_allowances(self) -> Dict[str, Decimal]: :return: A dictionary of token and its allowance (how much Balancer can spend). """ ret_val = {} - resp = await self._api_request("post", "eth/allowances-2", - {"tokenAddressList": ("".join([tok + "," for tok in self._token_addresses.values()])).rstrip(","), - "tokenDecimalList": ("".join([str(dec) + "," for dec in self._token_decimals.values()])).rstrip(","), + resp = await self._api_request("post", "eth/allowances", + {"tokenList": "[" + (",".join(['"' + t + '"' for t in self._tokens])) + "]", "connector": self.name}) - for address, amount in resp["approvals"].items(): - ret_val[self.get_token(address)] = Decimal(str(amount)) + for token, amount in resp["approvals"].items(): + ret_val[token] = Decimal(str(amount)) return ret_val @async_ttl_cache(ttl=5, maxsize=10) @@ -177,15 +185,44 @@ async def get_quote_price(self, trading_pair: str, is_buy: bool, amount: Decimal base, quote = trading_pair.split("-") side = "buy" if is_buy else "sell" resp = await self._api_request("post", - f"balancer/{side}-price", - {"base": self._token_addresses[base], - "quote": self._token_addresses[quote], + "eth/balancer/price", + {"base": base, + "quote": quote, "amount": amount, - "base_decimals": self._token_decimals[base], - "quote_decimals": self._token_decimals[quote], - "maxSwaps": self._max_swaps}) - if resp["price"] is not None: - return Decimal(str(resp["price"])) + "side": side.upper()}) + required_items = ["price", "gasLimit", "gasPrice", "gasCost"] + if any(item not in resp.keys() for item in required_items): + if "info" in resp.keys(): + self.logger().info(f"Unable to get price. {resp['info']}") + else: + self.logger().info(f"Missing data from price result. Incomplete return result for ({resp.keys()})") + else: + gas_limit = resp["gasLimit"] + gas_price = resp["gasPrice"] + gas_cost = resp["gasCost"] + price = resp["price"] + account_standing = { + "allowances": self._allowances, + "balances": self._account_balances, + "base": base, + "quote": quote, + "amount": amount, + "side": side, + "gas_limit": gas_limit, + "gas_price": gas_price, + "gas_cost": gas_cost, + "price": price, + "swaps": len(resp["swaps"]) + } + exceptions = check_transaction_exceptions(account_standing) + for index in range(len(exceptions)): + self.logger().info(f"Warning! [{index+1}/{len(exceptions)}] {side} order - {exceptions[index]}") + + if price is not None and len(exceptions) == 0: + # TODO standardize quote price object to include price, fee, token, is fee part of quote. + fee_overrides_config_map["balancer_maker_fee_amount"].value = Decimal(str(gas_cost)) + fee_overrides_config_map["balancer_taker_fee_amount"].value = Decimal(str(gas_cost)) + return Decimal(str(price)) except asyncio.CancelledError: raise except Exception as e: @@ -255,24 +292,28 @@ async def _create_order(self, amount = self.quantize_order_amount(trading_pair, amount) price = self.quantize_order_price(trading_pair, price) base, quote = trading_pair.split("-") - gas_price = get_gas_price() - api_params = {"base": self._token_addresses[base], - "quote": self._token_addresses[quote], + api_params = {"base": base, + "quote": quote, + "side": trade_type.name.upper(), "amount": str(amount), - "maxPrice": str(price), - "maxSwaps": str(self._max_swaps), - "gasPrice": str(gas_price), - "base_decimals": self._token_decimals[base], - "quote_decimals": self._token_decimals[quote], + "limitPrice": str(price), } - self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, gas_price) try: - order_result = await self._api_request("post", f"balancer/{trade_type.name.lower()}", api_params) + order_result = await self._api_request("post", "eth/balancer/trade", api_params) hash = order_result.get("txHash") + gas_price = order_result.get("gasPrice") + gas_limit = order_result.get("gasLimit") + gas_cost = order_result.get("gasCost") + self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, gas_price) tracked_order = self._in_flight_orders.get(order_id) + + # update onchain balance + await self._update_balances() + if tracked_order is not None: self.logger().info(f"Created {trade_type.name} order {order_id} txHash: {hash} " - f"for {amount} {trading_pair}.") + f"for {amount} {trading_pair}. Estimated Gas Cost: {gas_cost} ETH " + f" (gas limit: {gas_limit}, gas price: {gas_price})") tracked_order.update_exchange_order_id(hash) tracked_order.gas_price = gas_price if hash is not None: @@ -340,7 +381,7 @@ async def _update_order_status(self): for tracked_order in tracked_orders: order_id = await tracked_order.get_exchange_order_id() tasks.append(self._api_request("post", - "eth/get-receipt", + "eth/poll", {"txHash": order_id})) update_results = await safe_gather(*tasks, return_exceptions=True) for update_result in update_results: @@ -416,7 +457,7 @@ def has_allowances(self) -> bool: """ Checks if all tokens have allowance (an amount approved) """ - return len(self._allowances.values()) == len(self._token_addresses.values()) and \ + return len(self._allowances.values()) == len(self._tokens) and \ all(amount > s_decimal_0 for amount in self._allowances.values()) @property @@ -429,6 +470,7 @@ def status_dict(self) -> Dict[str, bool]: async def start_network(self): if self._trading_required: self._status_polling_task = safe_ensure_future(self._status_polling_loop()) + self._initiate_pool_task = safe_ensure_future(self.initiate_pool()) self._auto_approve_task = safe_ensure_future(self.auto_approve()) async def stop_network(self): @@ -438,6 +480,9 @@ async def stop_network(self): if self._auto_approve_task is not None: self._auto_approve_task.cancel() self._auto_approve_task = None + if self._initiate_pool_task is not None: + self._initiate_pool_task.cancel() + self._initiate_pool_task = None async def check_network(self) -> NetworkStatus: try: @@ -478,9 +523,6 @@ async def _status_polling_loop(self): app_warning_msg="Could not fetch balances from Gateway API.") await asyncio.sleep(0.5) - def get_token(self, token_address: str) -> str: - return [k for k, v in self._token_addresses.items() if v == token_address][0] - async def _update_balances(self, on_interval = False): """ Calls Eth API to update total and available balances. @@ -492,12 +534,10 @@ async def _update_balances(self, on_interval = False): local_asset_names = set(self._account_balances.keys()) remote_asset_names = set() resp_json = await self._api_request("post", - "eth/balances-2", - {"tokenAddressList": ("".join([tok + "," for tok in self._token_addresses.values()])).rstrip(","), - "tokenDecimalList": ("".join([str(dec) + "," for dec in self._token_decimals.values()])).rstrip(",")}) + "eth/balances", + {"tokenList": "[" + (",".join(['"' + t + '"' for t in self._tokens])) + "]"}) + for token, bal in resp_json["balances"].items(): - if len(token) > 4: - token = self.get_token(token) self._account_available_balances[token] = Decimal(str(bal)) self._account_balances[token] = Decimal(str(bal)) remote_asset_names.add(token) @@ -554,7 +594,7 @@ async def _api_request(self, err_msg = f" Message: {parsed_response['error']}" raise IOError(f"Error fetching data from {url}. HTTP status is {response.status}.{err_msg}") if "error" in parsed_response: - raise Exception(f"Error: {parsed_response['error']}") + raise Exception(f"Error: {parsed_response['error']} {parsed_response['message']}") return parsed_response diff --git a/hummingbot/connector/connector/terra/terra_connector.py b/hummingbot/connector/connector/terra/terra_connector.py index 1836750621..0f27b3e0ed 100644 --- a/hummingbot/connector/connector/terra/terra_connector.py +++ b/hummingbot/connector/connector/terra/terra_connector.py @@ -107,7 +107,7 @@ async def get_quote_price(self, trading_pair: str, is_buy: bool, amount: Decimal base, quote = trading_pair.split("-") side = "buy" if is_buy else "sell" - resp = await self._api_request("post", "terra/price", {"base": base, "quote": quote, "trade_type": side, + resp = await self._api_request("post", "terra/price", {"base": base, "quote": quote, "side": side, "amount": str(amount)}) txFee = resp["txFee"] / float(amount) price_with_txfee = resp["price"] + txFee if is_buy else resp["price"] - txFee @@ -185,9 +185,9 @@ async def _create_order(self, base, quote = trading_pair.split("-") api_params = {"base": base, "quote": quote, - "trade_type": "buy" if trade_type is TradeType.BUY else "sell", + "side": "buy" if trade_type is TradeType.BUY else "sell", "amount": str(amount), - "secret": self._terra_wallet_seeds, + "privateKey": self._terra_wallet_seeds, # "maxPrice": str(price), } self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount) diff --git a/hummingbot/connector/connector/uniswap/uniswap_connector.py b/hummingbot/connector/connector/uniswap/uniswap_connector.py index 2fef254945..c8277b65b1 100644 --- a/hummingbot/connector/connector/uniswap/uniswap_connector.py +++ b/hummingbot/connector/connector/uniswap/uniswap_connector.py @@ -7,7 +7,6 @@ import time import ssl import copy -import itertools as it from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL from hummingbot.core.utils import async_ttl_cache from hummingbot.core.network_iterator import NetworkStatus @@ -31,9 +30,9 @@ from hummingbot.connector.connector_base import ConnectorBase from hummingbot.connector.connector.uniswap.uniswap_in_flight_order import UniswapInFlightOrder from hummingbot.client.settings import GATEAWAY_CA_CERT_PATH, GATEAWAY_CLIENT_CERT_PATH, GATEAWAY_CLIENT_KEY_PATH -from hummingbot.core.utils.eth_gas_station_lookup import get_gas_price from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.client.config.config_helpers import get_erc20_token_addresses +from hummingbot.core.utils.ethereum import check_transaction_exceptions, fetch_trading_pairs +from hummingbot.client.config.fee_overrides_config_map import fee_overrides_config_map s_logger = None s_decimal_0 = Decimal("0") @@ -71,12 +70,9 @@ def __init__(self, """ super().__init__() self._trading_pairs = trading_pairs - tokens = set() + self._tokens = set() for trading_pair in trading_pairs: - tokens.update(set(trading_pair.split("-"))) - self._erc_20_token_list = self.token_list() - self._token_addresses = {t: l[0] for t, l in self._erc_20_token_list.items() if t in tokens} - self._token_decimals = {t: l[1] for t, l in self._erc_20_token_list.items() if t in tokens} + self._tokens.update(set(trading_pair.split("-"))) self._wallet_private_key = wallet_private_key self._ethereum_rpc_url = ethereum_rpc_url self._trading_required = trading_required @@ -84,10 +80,13 @@ def __init__(self, self._shared_client = None self._last_poll_timestamp = 0.0 self._last_balance_poll_timestamp = time.time() + self._last_est_gas_cost_reported = 0 self._in_flight_orders = {} self._allowances = {} self._status_polling_task = None self._auto_approve_task = None + self._initiate_pool_task = None + self._initiate_pool_status = None self._real_time_balance_update = False self._poll_notifier = None @@ -95,17 +94,9 @@ def __init__(self, def name(self): return "uniswap" - @staticmethod - def token_list(): - return get_erc20_token_addresses() - @staticmethod async def fetch_trading_pairs() -> List[str]: - token_list = UniswapConnector.token_list() - trading_pairs = [] - for base, quote in it.permutations(token_list.keys(), 2): - trading_pairs.append(f"{base}-{quote}") - return trading_pairs + return await fetch_trading_pairs() @property def limit_orders(self) -> List[LimitOrder]: @@ -114,6 +105,26 @@ def limit_orders(self) -> List[LimitOrder]: for in_flight_order in self._in_flight_orders.values() ] + async def initiate_pool(self) -> str: + """ + Initiate connector and start caching paths for trading_pairs + """ + try: + self.logger().info(f"Initializing Uniswap connector and paths for {self._trading_pairs} pairs.") + resp = await self._api_request("get", "eth/uniswap/start", + {"pairs": json.dumps(self._trading_pairs)}) + status = bool(str(resp["success"])) + if bool(str(resp["success"])): + self._initiate_pool_status = status + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().network( + f"Error initializing {self._trading_pairs} ", + exc_info=True, + app_warning_msg=str(e) + ) + async def auto_approve(self): """ Automatically approves Uniswap contract as a spender for token in trading pairs. @@ -137,9 +148,7 @@ async def approve_uniswap_spender(self, token_symbol: str) -> Decimal: """ resp = await self._api_request("post", "eth/approve", - {"tokenAddress": self._token_addresses[token_symbol], - "gasPrice": str(get_gas_price()), - "decimals": self._token_decimals[token_symbol], # if not supplied, gateway would treat it eth-like with 18 decimals + {"token": token_symbol, "connector": self.name}) amount_approved = Decimal(str(resp["amount"])) if amount_approved > 0: @@ -154,12 +163,11 @@ async def get_allowances(self) -> Dict[str, Decimal]: :return: A dictionary of token and its allowance (how much Uniswap can spend). """ ret_val = {} - resp = await self._api_request("post", "eth/allowances-2", - {"tokenAddressList": ("".join([tok + "," for tok in self._token_addresses.values()])).rstrip(","), - "tokenDecimalList": ("".join([str(dec) + "," for dec in self._token_decimals.values()])).rstrip(","), + resp = await self._api_request("post", "eth/allowances", + {"tokenList": "[" + (",".join(['"' + t + '"' for t in self._tokens])) + "]", "connector": self.name}) - for address, amount in resp["approvals"].items(): - ret_val[self.get_token(address)] = Decimal(str(amount)) + for token, amount in resp["approvals"].items(): + ret_val[token] = Decimal(str(amount)) return ret_val @async_ttl_cache(ttl=5, maxsize=10) @@ -176,12 +184,43 @@ async def get_quote_price(self, trading_pair: str, is_buy: bool, amount: Decimal base, quote = trading_pair.split("-") side = "buy" if is_buy else "sell" resp = await self._api_request("post", - f"uniswap/{side}-price", - {"base": self._token_addresses[base], - "quote": self._token_addresses[quote], + "eth/uniswap/price", + {"base": base, + "quote": quote, + "side": side.upper(), "amount": amount}) - if resp["price"] is not None: - return Decimal(str(resp["price"])) + required_items = ["price", "gasLimit", "gasPrice", "gasCost"] + if any(item not in resp.keys() for item in required_items): + if "info" in resp.keys(): + self.logger().info(f"Unable to get price. {resp['info']}") + else: + self.logger().info(f"Missing data from price result. Incomplete return result for ({resp.keys()})") + else: + gas_limit = resp["gasLimit"] + gas_price = resp["gasPrice"] + gas_cost = resp["gasCost"] + price = resp["price"] + account_standing = { + "allowances": self._allowances, + "balances": self._account_balances, + "base": base, + "quote": quote, + "amount": amount, + "side": side, + "gas_limit": gas_limit, + "gas_price": gas_price, + "gas_cost": gas_cost, + "price": price + } + exceptions = check_transaction_exceptions(account_standing) + for index in range(len(exceptions)): + self.logger().info(f"Warning! [{index+1}/{len(exceptions)}] {side} order - {exceptions[index]}") + + if price is not None and len(exceptions) == 0: + # TODO standardize quote price object to include price, fee, token, is fee part of quote. + fee_overrides_config_map["uniswap_maker_fee_amount"].value = Decimal(str(gas_cost)) + fee_overrides_config_map["uniswap_taker_fee_amount"].value = Decimal(str(gas_cost)) + return Decimal(str(price)) except asyncio.CancelledError: raise except Exception as e: @@ -251,21 +290,24 @@ async def _create_order(self, amount = self.quantize_order_amount(trading_pair, amount) price = self.quantize_order_price(trading_pair, price) base, quote = trading_pair.split("-") - gas_price = get_gas_price() - api_params = {"base": self._token_addresses[base], - "quote": self._token_addresses[quote], + api_params = {"base": base, + "quote": quote, + "side": trade_type.name.upper(), "amount": str(amount), - "maxPrice": str(price), - "gasPrice": str(gas_price), + "limitPrice": str(price), } - self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, gas_price) try: - order_result = await self._api_request("post", f"uniswap/{trade_type.name.lower()}", api_params) + order_result = await self._api_request("post", "eth/uniswap/trade", api_params) hash = order_result.get("txHash") + gas_price = order_result.get("gasPrice") + gas_limit = order_result.get("gasLimit") + gas_cost = order_result.get("gasCost") + self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, gas_price) tracked_order = self._in_flight_orders.get(order_id) if tracked_order is not None: self.logger().info(f"Created {trade_type.name} order {order_id} txHash: {hash} " - f"for {amount} {trading_pair}.") + f"for {amount} {trading_pair}. Estimated Gas Cost: {gas_cost} ETH " + f" (gas limit: {gas_limit}, gas price: {gas_price})") tracked_order.update_exchange_order_id(hash) tracked_order.gas_price = gas_price if hash is not None: @@ -333,7 +375,7 @@ async def _update_order_status(self): for tracked_order in tracked_orders: order_id = await tracked_order.get_exchange_order_id() tasks.append(self._api_request("post", - "eth/get-receipt", + "eth/poll", {"txHash": order_id})) update_results = await safe_gather(*tasks, return_exceptions=True) for update_result in update_results: @@ -409,7 +451,7 @@ def has_allowances(self) -> bool: """ Checks if all tokens have allowance (an amount approved) """ - return len(self._allowances.values()) == len(self._token_addresses.values()) and \ + return len(self._allowances.values()) == len(self._tokens) and \ all(amount > s_decimal_0 for amount in self._allowances.values()) @property @@ -422,6 +464,7 @@ def status_dict(self) -> Dict[str, bool]: async def start_network(self): if self._trading_required: self._status_polling_task = safe_ensure_future(self._status_polling_loop()) + self._initiate_pool_task = safe_ensure_future(self.initiate_pool()) self._auto_approve_task = safe_ensure_future(self.auto_approve()) async def stop_network(self): @@ -431,6 +474,9 @@ async def stop_network(self): if self._auto_approve_task is not None: self._auto_approve_task.cancel() self._auto_approve_task = None + if self._initiate_pool_task is not None: + self._initiate_pool_task.cancel() + self._initiate_pool_task = None async def check_network(self) -> NetworkStatus: try: @@ -458,7 +504,7 @@ async def _status_polling_loop(self): self._poll_notifier = asyncio.Event() await self._poll_notifier.wait() await safe_gather( - self._update_balances(), + self._update_balances(on_interval=True), self._update_order_status(), ) self._last_poll_timestamp = self.current_timestamp @@ -471,37 +517,32 @@ async def _status_polling_loop(self): app_warning_msg="Could not fetch balances from Gateway API.") await asyncio.sleep(0.5) - def get_token(self, token_address: str) -> str: - return [k for k, v in self._token_addresses.items() if v == token_address][0] - - async def _update_balances(self): + async def _update_balances(self, on_interval = False): """ Calls Eth API to update total and available balances. """ last_tick = self._last_balance_poll_timestamp current_tick = self.current_timestamp - if (current_tick - last_tick) > self.UPDATE_BALANCE_INTERVAL: + if not on_interval or (current_tick - last_tick) > self.UPDATE_BALANCE_INTERVAL: self._last_balance_poll_timestamp = current_tick - local_asset_names = set(self._account_balances.keys()) - remote_asset_names = set() - resp_json = await self._api_request("post", - "eth/balances-2", - {"tokenAddressList": ("".join([tok + "," for tok in self._token_addresses.values()])).rstrip(","), - "tokenDecimalList": ("".join([str(dec) + "," for dec in self._token_decimals.values()])).rstrip(",")}) - for token, bal in resp_json["balances"].items(): - if len(token) > 4: - token = self.get_token(token) - self._account_available_balances[token] = Decimal(str(bal)) - self._account_balances[token] = Decimal(str(bal)) - remote_asset_names.add(token) - - asset_names_to_remove = local_asset_names.difference(remote_asset_names) - for asset_name in asset_names_to_remove: - del self._account_available_balances[asset_name] - del self._account_balances[asset_name] - - self._in_flight_orders_snapshot = {k: copy.copy(v) for k, v in self._in_flight_orders.items()} - self._in_flight_orders_snapshot_timestamp = self.current_timestamp + local_asset_names = set(self._account_balances.keys()) + remote_asset_names = set() + resp_json = await self._api_request("post", + "eth/balances", + {"tokenList": "[" + (",".join(['"' + t + '"' for t in self._tokens])) + "]"}) + + for token, bal in resp_json["balances"].items(): + self._account_available_balances[token] = Decimal(str(bal)) + self._account_balances[token] = Decimal(str(bal)) + remote_asset_names.add(token) + + asset_names_to_remove = local_asset_names.difference(remote_asset_names) + for asset_name in asset_names_to_remove: + del self._account_available_balances[asset_name] + del self._account_balances[asset_name] + + self._in_flight_orders_snapshot = {k: copy.copy(v) for k, v in self._in_flight_orders.items()} + self._in_flight_orders_snapshot_timestamp = self.current_timestamp async def _http_client(self) -> aiohttp.ClientSession: """ @@ -547,7 +588,7 @@ async def _api_request(self, err_msg = f" Message: {parsed_response['error']}" raise IOError(f"Error fetching data from {url}. HTTP status is {response.status}.{err_msg}") if "error" in parsed_response: - raise Exception(f"Error: {parsed_response['error']}") + raise Exception(f"Error: {parsed_response['error']} {parsed_response['message']}") return parsed_response diff --git a/hummingbot/connector/connector_status.py b/hummingbot/connector/connector_status.py index f220808b1e..14a0968094 100644 --- a/hummingbot/connector/connector_status.py +++ b/hummingbot/connector/connector_status.py @@ -8,7 +8,7 @@ 'binance_perpetual_testnet': 'green', 'binance_us': 'yellow', 'bitfinex': 'yellow', - 'bitmax': 'green', + 'ascend_ex': 'green', 'bittrex': 'yellow', 'blocktane': 'green', 'celo': 'green', @@ -17,6 +17,7 @@ 'dydx': 'green', 'eterbase': 'red', 'ethereum': 'red', + 'hitbtc': 'yellow', 'huobi': 'green', 'kraken': 'green', 'kucoin': 'green', diff --git a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_api_order_book_data_source.py b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_api_order_book_data_source.py index b6e54ed720..9dc900251f 100644 --- a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_api_order_book_data_source.py +++ b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_api_order_book_data_source.py @@ -2,13 +2,10 @@ import logging import time from typing import Dict, List, Optional, Any, AsyncIterable -from decimal import Decimal import aiohttp import pandas as pd import ujson -import requests -import cachetools.func import websockets from websockets.exceptions import ConnectionClosed @@ -83,17 +80,6 @@ async def get_trading_pairs(self) -> List[str]: return self._trading_pairs """ - @staticmethod - @cachetools.func.ttl_cache(ttl=10) - def get_mid_price(trading_pair: str, domain=None) -> Optional[Decimal]: - from hummingbot.connector.derivative.binance_perpetual.binance_perpetual_utils import convert_to_exchange_trading_pair - - BASE_URL = TESTNET_BASE_URL if domain == "binance_perpetual_testnet" else PERPETUAL_BASE_URL - resp = requests.get(url=f"{TICKER_PRICE_URL.format(BASE_URL)}?symbol={convert_to_exchange_trading_pair(trading_pair)}") - record = resp.json() - result = (Decimal(record["bidPrice"]) + Decimal(record["askPrice"])) / Decimal("2") - return result - @staticmethod async def fetch_trading_pairs(domain=None) -> List[str]: try: diff --git a/hummingbot/connector/exchange/bitmax/__init__.py b/hummingbot/connector/exchange/ascend_ex/__init__.py similarity index 100% rename from hummingbot/connector/exchange/bitmax/__init__.py rename to hummingbot/connector/exchange/ascend_ex/__init__.py diff --git a/hummingbot/connector/exchange/bitmax/bitmax_active_order_tracker.pxd b/hummingbot/connector/exchange/ascend_ex/ascend_ex_active_order_tracker.pxd similarity index 90% rename from hummingbot/connector/exchange/bitmax/bitmax_active_order_tracker.pxd rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_active_order_tracker.pxd index fbc1eb3080..833d9864a0 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_active_order_tracker.pxd +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_active_order_tracker.pxd @@ -1,7 +1,7 @@ # distutils: language=c++ cimport numpy as np -cdef class BitmaxActiveOrderTracker: +cdef class AscendExActiveOrderTracker: cdef dict _active_bids cdef dict _active_asks diff --git a/hummingbot/connector/exchange/bitmax/bitmax_active_order_tracker.pyx b/hummingbot/connector/exchange/ascend_ex/ascend_ex_active_order_tracker.pyx similarity index 93% rename from hummingbot/connector/exchange/bitmax/bitmax_active_order_tracker.pyx rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_active_order_tracker.pyx index 092a97c45c..b80930b8b2 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_active_order_tracker.pyx +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_active_order_tracker.pyx @@ -12,12 +12,12 @@ from hummingbot.core.data_type.order_book_row import OrderBookRow _logger = None s_empty_diff = np.ndarray(shape=(0, 4), dtype="float64") -BitmaxOrderBookTrackingDictionary = Dict[Decimal, Dict[str, Dict[str, any]]] +AscendExOrderBookTrackingDictionary = Dict[Decimal, Dict[str, Dict[str, any]]] -cdef class BitmaxActiveOrderTracker: +cdef class AscendExActiveOrderTracker: def __init__(self, - active_asks: BitmaxOrderBookTrackingDictionary = None, - active_bids: BitmaxOrderBookTrackingDictionary = None): + active_asks: AscendExOrderBookTrackingDictionary = None, + active_bids: AscendExOrderBookTrackingDictionary = None): super().__init__() self._active_asks = active_asks or {} self._active_bids = active_bids or {} @@ -30,11 +30,11 @@ cdef class BitmaxActiveOrderTracker: return _logger @property - def active_asks(self) -> BitmaxOrderBookTrackingDictionary: + def active_asks(self) -> AscendExOrderBookTrackingDictionary: return self._active_asks @property - def active_bids(self) -> BitmaxOrderBookTrackingDictionary: + def active_bids(self) -> AscendExOrderBookTrackingDictionary: return self._active_bids # TODO: research this more diff --git a/hummingbot/connector/exchange/bitmax/bitmax_api_order_book_data_source.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_api_order_book_data_source.py similarity index 91% rename from hummingbot/connector/exchange/bitmax/bitmax_api_order_book_data_source.py rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_api_order_book_data_source.py index 306b0631c4..0c64a9c1c4 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_api_order_book_data_source.py @@ -13,13 +13,13 @@ from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource from hummingbot.core.utils.async_utils import safe_gather from hummingbot.logger import HummingbotLogger -from hummingbot.connector.exchange.bitmax.bitmax_active_order_tracker import BitmaxActiveOrderTracker -from hummingbot.connector.exchange.bitmax.bitmax_order_book import BitmaxOrderBook -from hummingbot.connector.exchange.bitmax.bitmax_utils import convert_from_exchange_trading_pair, convert_to_exchange_trading_pair -from hummingbot.connector.exchange.bitmax.bitmax_constants import EXCHANGE_NAME, REST_URL, WS_URL, PONG_PAYLOAD +from hummingbot.connector.exchange.ascend_ex.ascend_ex_active_order_tracker import AscendExActiveOrderTracker +from hummingbot.connector.exchange.ascend_ex.ascend_ex_order_book import AscendExOrderBook +from hummingbot.connector.exchange.ascend_ex.ascend_ex_utils import convert_from_exchange_trading_pair, convert_to_exchange_trading_pair +from hummingbot.connector.exchange.ascend_ex.ascend_ex_constants import EXCHANGE_NAME, REST_URL, WS_URL, PONG_PAYLOAD -class BitmaxAPIOrderBookDataSource(OrderBookTrackerDataSource): +class AscendExAPIOrderBookDataSource(OrderBookTrackerDataSource): MAX_RETRIES = 20 MESSAGE_TIMEOUT = 30.0 SNAPSHOT_TIMEOUT = 10.0 @@ -105,13 +105,13 @@ async def get_order_book_data(trading_pair: str) -> Dict[str, any]: async def get_new_order_book(self, trading_pair: str) -> OrderBook: snapshot: Dict[str, Any] = await self.get_order_book_data(trading_pair) snapshot_timestamp: float = snapshot.get("data").get("ts") - snapshot_msg: OrderBookMessage = BitmaxOrderBook.snapshot_message_from_exchange( + snapshot_msg: OrderBookMessage = AscendExOrderBook.snapshot_message_from_exchange( snapshot.get("data"), snapshot_timestamp, metadata={"trading_pair": trading_pair} ) order_book = self.order_book_create_function() - active_order_tracker: BitmaxActiveOrderTracker = BitmaxActiveOrderTracker() + active_order_tracker: AscendExActiveOrderTracker = AscendExActiveOrderTracker() bids, asks = active_order_tracker.convert_snapshot_message_to_order_book_row(snapshot_msg) order_book.apply_snapshot(bids, asks, snapshot_msg.update_id) return order_book @@ -141,7 +141,7 @@ async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asynci for trade in msg.get("data"): trade_timestamp: int = trade.get("ts") - trade_msg: OrderBookMessage = BitmaxOrderBook.trade_message_from_exchange( + trade_msg: OrderBookMessage = AscendExOrderBook.trade_message_from_exchange( trade, trade_timestamp, metadata={"trading_pair": trading_pair} @@ -181,7 +181,7 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp msg_timestamp: int = msg.get("data").get("ts") trading_pair: str = convert_from_exchange_trading_pair(msg.get("symbol")) - order_book_message: OrderBookMessage = BitmaxOrderBook.diff_message_from_exchange( + order_book_message: OrderBookMessage = AscendExOrderBook.diff_message_from_exchange( msg.get("data"), msg_timestamp, metadata={"trading_pair": trading_pair} @@ -207,7 +207,7 @@ async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, try: snapshot: Dict[str, any] = await self.get_order_book_data(trading_pair) snapshot_timestamp: float = snapshot.get("data").get("ts") - snapshot_msg: OrderBookMessage = BitmaxOrderBook.snapshot_message_from_exchange( + snapshot_msg: OrderBookMessage = AscendExOrderBook.snapshot_message_from_exchange( snapshot.get("data"), snapshot_timestamp, metadata={"trading_pair": trading_pair} diff --git a/hummingbot/connector/exchange/bitmax/bitmax_api_user_stream_data_source.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_api_user_stream_data_source.py similarity index 79% rename from hummingbot/connector/exchange/bitmax/bitmax_api_user_stream_data_source.py rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_api_user_stream_data_source.py index 10bbac6763..19571d90c8 100755 --- a/hummingbot/connector/exchange/bitmax/bitmax_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_api_user_stream_data_source.py @@ -9,11 +9,12 @@ from typing import Optional, List, AsyncIterable, Any from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource from hummingbot.logger import HummingbotLogger -from hummingbot.connector.exchange.bitmax.bitmax_auth import BitmaxAuth -from hummingbot.connector.exchange.bitmax.bitmax_constants import REST_URL, getWsUrlPriv, PONG_PAYLOAD +from hummingbot.connector.exchange.ascend_ex.ascend_ex_auth import AscendExAuth +from hummingbot.connector.exchange.ascend_ex.ascend_ex_constants import REST_URL, PONG_PAYLOAD +from hummingbot.connector.exchange.ascend_ex.ascend_ex_utils import get_ws_url_private -class BitmaxAPIUserStreamDataSource(UserStreamTrackerDataSource): +class AscendExAPIUserStreamDataSource(UserStreamTrackerDataSource): MAX_RETRIES = 20 MESSAGE_TIMEOUT = 10.0 PING_TIMEOUT = 5.0 @@ -26,8 +27,8 @@ def logger(cls) -> HummingbotLogger: cls._logger = logging.getLogger(__name__) return cls._logger - def __init__(self, bitmax_auth: BitmaxAuth, trading_pairs: Optional[List[str]] = []): - self._bitmax_auth: BitmaxAuth = bitmax_auth + def __init__(self, ascend_ex_auth: AscendExAuth, trading_pairs: Optional[List[str]] = []): + self._ascend_ex_auth: AscendExAuth = ascend_ex_auth self._trading_pairs = trading_pairs self._current_listen_key = None self._listen_for_user_stream_task = None @@ -50,18 +51,18 @@ async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: a while True: try: response = await aiohttp.ClientSession().get(f"{REST_URL}/info", headers={ - **self._bitmax_auth.get_headers(), - **self._bitmax_auth.get_auth_headers("info"), + **self._ascend_ex_auth.get_headers(), + **self._ascend_ex_auth.get_auth_headers("info"), }) info = await response.json() accountGroup = info.get("data").get("accountGroup") - headers = self._bitmax_auth.get_auth_headers("stream") + headers = self._ascend_ex_auth.get_auth_headers("stream") payload = { "op": "sub", "ch": "order:cash" } - async with websockets.connect(f"{getWsUrlPriv(accountGroup)}/stream", extra_headers=headers) as ws: + async with websockets.connect(f"{get_ws_url_private(accountGroup)}/stream", extra_headers=headers) as ws: try: ws: websockets.WebSocketClientProtocol = ws await ws.send(ujson.dumps(payload)) @@ -75,12 +76,12 @@ async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: a output.put_nowait(msg) except Exception: self.logger().error( - "Unexpected error when parsing Bitmax message. ", exc_info=True + "Unexpected error when parsing AscendEx message. ", exc_info=True ) raise except Exception: self.logger().error( - "Unexpected error while listening to Bitmax messages. ", exc_info=True + "Unexpected error while listening to AscendEx messages. ", exc_info=True ) raise finally: @@ -89,7 +90,7 @@ async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: a raise except Exception: self.logger().error( - "Unexpected error with Bitmax WebSocket connection. " "Retrying after 30 seconds...", exc_info=True + "Unexpected error with AscendEx WebSocket connection. " "Retrying after 30 seconds...", exc_info=True ) await asyncio.sleep(30.0) diff --git a/hummingbot/connector/exchange/bitmax/bitmax_auth.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_auth.py similarity index 79% rename from hummingbot/connector/exchange/bitmax/bitmax_auth.py rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_auth.py index 646fb13a8a..23b2c78f67 100755 --- a/hummingbot/connector/exchange/bitmax/bitmax_auth.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_auth.py @@ -1,13 +1,13 @@ import hmac import hashlib from typing import Dict, Any -from hummingbot.connector.exchange.bitmax.bitmax_utils import get_ms_timestamp +from hummingbot.connector.exchange.ascend_ex.ascend_ex_utils import get_ms_timestamp -class BitmaxAuth(): +class AscendExAuth(): """ - Auth class required by bitmax API - Learn more at https://bitmax-exchange.github.io/bitmax-pro-api/#authenticate-a-restful-request + Auth class required by AscendEx API + Learn more at https://ascendex.github.io/ascendex-pro-api/#authenticate-a-restful-request """ def __init__(self, api_key: str, secret_key: str): self.api_key = api_key @@ -39,7 +39,7 @@ def get_auth_headers( def get_headers(self) -> Dict[str, Any]: """ - Generates generic headers required by bitmax + Generates generic headers required by AscendEx :return: a dictionary of headers """ diff --git a/hummingbot/connector/exchange/ascend_ex/ascend_ex_constants.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_constants.py new file mode 100644 index 0000000000..27165b3257 --- /dev/null +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_constants.py @@ -0,0 +1,7 @@ +# A single source of truth for constant variables related to the exchange + + +EXCHANGE_NAME = "ascend_ex" +REST_URL = "https://ascendex.com/api/pro/v1" +WS_URL = "wss://ascendex.com/0/api/pro/v1/stream" +PONG_PAYLOAD = {"op": "pong"} diff --git a/hummingbot/connector/exchange/bitmax/bitmax_exchange.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_exchange.py similarity index 88% rename from hummingbot/connector/exchange/bitmax/bitmax_exchange.py rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_exchange.py index 4606157dd3..a8aac4e618 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_exchange.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_exchange.py @@ -36,25 +36,25 @@ TradeFee ) from hummingbot.connector.exchange_base import ExchangeBase -from hummingbot.connector.exchange.bitmax.bitmax_order_book_tracker import BitmaxOrderBookTracker -from hummingbot.connector.exchange.bitmax.bitmax_user_stream_tracker import BitmaxUserStreamTracker -from hummingbot.connector.exchange.bitmax.bitmax_auth import BitmaxAuth -from hummingbot.connector.exchange.bitmax.bitmax_in_flight_order import BitmaxInFlightOrder -from hummingbot.connector.exchange.bitmax import bitmax_utils -from hummingbot.connector.exchange.bitmax.bitmax_constants import EXCHANGE_NAME, REST_URL, getRestUrlPriv +from hummingbot.connector.exchange.ascend_ex.ascend_ex_order_book_tracker import AscendExOrderBookTracker +from hummingbot.connector.exchange.ascend_ex.ascend_ex_user_stream_tracker import AscendExUserStreamTracker +from hummingbot.connector.exchange.ascend_ex.ascend_ex_auth import AscendExAuth +from hummingbot.connector.exchange.ascend_ex.ascend_ex_in_flight_order import AscendExInFlightOrder +from hummingbot.connector.exchange.ascend_ex import ascend_ex_utils +from hummingbot.connector.exchange.ascend_ex.ascend_ex_constants import EXCHANGE_NAME, REST_URL from hummingbot.core.data_type.common import OpenOrder ctce_logger = None s_decimal_NaN = Decimal("nan") -BitmaxTradingRule = namedtuple("BitmaxTradingRule", "minNotional maxNotional") -BitmaxOrder = namedtuple("BitmaxOrder", "symbol price orderQty orderType avgPx cumFee cumFilledQty errorCode feeAsset lastExecTime orderId seqNum side status stopPrice execInst") -BitmaxBalance = namedtuple("BitmaxBalance", "asset availableBalance totalBalance") +AscendExTradingRule = namedtuple("AscendExTradingRule", "minNotional maxNotional") +AscendExOrder = namedtuple("AscendExOrder", "symbol price orderQty orderType avgPx cumFee cumFilledQty errorCode feeAsset lastExecTime orderId seqNum side status stopPrice execInst") +AscendExBalance = namedtuple("AscendExBalance", "asset availableBalance totalBalance") -class BitmaxExchange(ExchangeBase): +class AscendExExchange(ExchangeBase): """ - BitmaxExchange connects with Bitmax exchange and provides order book pricing, user account tracking and + AscendExExchange connects with AscendEx exchange and provides order book pricing, user account tracking and trading functionality. """ API_CALL_TIMEOUT = 10.0 @@ -70,31 +70,31 @@ def logger(cls) -> HummingbotLogger: return ctce_logger def __init__(self, - bitmax_api_key: str, - bitmax_secret_key: str, + ascend_ex_api_key: str, + ascend_ex_secret_key: str, trading_pairs: Optional[List[str]] = None, trading_required: bool = True ): """ - :param bitmax_api_key: The API key to connect to private Bitmax APIs. - :param bitmax_secret_key: The API secret. + :param ascend_ex_api_key: The API key to connect to private AscendEx APIs. + :param ascend_ex_secret_key: The API secret. :param trading_pairs: The market trading pairs which to track order book data. :param trading_required: Whether actual trading is needed. """ super().__init__() self._trading_required = trading_required self._trading_pairs = trading_pairs - self._bitmax_auth = BitmaxAuth(bitmax_api_key, bitmax_secret_key) - self._order_book_tracker = BitmaxOrderBookTracker(trading_pairs=trading_pairs) - self._user_stream_tracker = BitmaxUserStreamTracker(self._bitmax_auth, trading_pairs) + self._ascend_ex_auth = AscendExAuth(ascend_ex_api_key, ascend_ex_secret_key) + self._order_book_tracker = AscendExOrderBookTracker(trading_pairs=trading_pairs) + self._user_stream_tracker = AscendExUserStreamTracker(self._ascend_ex_auth, trading_pairs) self._ev_loop = asyncio.get_event_loop() self._shared_client = None self._poll_notifier = asyncio.Event() self._last_timestamp = 0 - self._in_flight_orders = {} # Dict[client_order_id:str, BitmaxInFlightOrder] + self._in_flight_orders = {} # Dict[client_order_id:str, AscendExInFlightOrder] self._order_not_found_records = {} # Dict[client_order_id:str, count:int] self._trading_rules = {} # Dict[trading_pair:str, TradingRule] - self._bitmax_trading_rules = {} # Dict[trading_pair:str, BitmaxTradingRule] + self._ascend_ex_trading_rules = {} # Dict[trading_pair:str, AscendExTradingRule] self._status_polling_task = None self._user_stream_event_listener_task = None self._trading_rules_polling_task = None @@ -115,7 +115,7 @@ def trading_rules(self) -> Dict[str, TradingRule]: return self._trading_rules @property - def in_flight_orders(self) -> Dict[str, BitmaxInFlightOrder]: + def in_flight_orders(self) -> Dict[str, AscendExInFlightOrder]: return self._in_flight_orders @property @@ -126,7 +126,7 @@ def status_dict(self) -> Dict[str, bool]: return { "order_books_initialized": self._order_book_tracker.ready, "account_balance": len(self._account_balances) > 0 if self._trading_required else True, - "trading_rule_initialized": len(self._trading_rules) > 0 and len(self._bitmax_trading_rules) > 0, + "trading_rule_initialized": len(self._trading_rules) > 0 and len(self._ascend_ex_trading_rules) > 0, "user_stream_initialized": self._user_stream_tracker.data_source.last_recv_time > 0 if self._trading_required else True, "account_data": self._account_group is not None and self._account_uid is not None @@ -165,7 +165,7 @@ def restore_tracking_states(self, saved_states: Dict[str, any]): :param saved_states: The saved tracking_states. """ self._in_flight_orders.update({ - key: BitmaxInFlightOrder.from_json(value) + key: AscendExInFlightOrder.from_json(value) for key, value in saved_states.items() }) @@ -258,22 +258,22 @@ async def _trading_rules_polling_loop(self): except Exception as e: self.logger().network(f"Unexpected error while fetching trading rules. Error: {str(e)}", exc_info=True, - app_warning_msg="Could not fetch new trading rules from Bitmax. " + app_warning_msg="Could not fetch new trading rules from AscendEx. " "Check network connection.") await asyncio.sleep(0.5) async def _update_trading_rules(self): instruments_info = await self._api_request("get", path_url="products") - [trading_rules, bitmax_trading_rules] = self._format_trading_rules(instruments_info) + [trading_rules, ascend_ex_trading_rules] = self._format_trading_rules(instruments_info) self._trading_rules.clear() self._trading_rules = trading_rules - self._bitmax_trading_rules.clear() - self._bitmax_trading_rules = bitmax_trading_rules + self._ascend_ex_trading_rules.clear() + self._ascend_ex_trading_rules = ascend_ex_trading_rules def _format_trading_rules( self, instruments_info: Dict[str, Any] - ) -> [Dict[str, TradingRule], Dict[str, Dict[str, BitmaxTradingRule]]]: + ) -> [Dict[str, TradingRule], Dict[str, Dict[str, AscendExTradingRule]]]: """ Converts json API response into a dictionary of trading rules. :param instruments_info: The json API response @@ -299,27 +299,27 @@ def _format_trading_rules( } """ trading_rules = {} - bitmax_trading_rules = {} + ascend_ex_trading_rules = {} for rule in instruments_info["data"]: try: - trading_pair = bitmax_utils.convert_from_exchange_trading_pair(rule["symbol"]) + trading_pair = ascend_ex_utils.convert_from_exchange_trading_pair(rule["symbol"]) trading_rules[trading_pair] = TradingRule( trading_pair, min_price_increment=Decimal(rule["tickSize"]), min_base_amount_increment=Decimal(rule["lotSize"]) ) - bitmax_trading_rules[trading_pair] = BitmaxTradingRule( + ascend_ex_trading_rules[trading_pair] = AscendExTradingRule( minNotional=Decimal(rule["minNotional"]), maxNotional=Decimal(rule["maxNotional"]) ) except Exception: self.logger().error(f"Error parsing the trading pair rule {rule}. Skipping.", exc_info=True) - return [trading_rules, bitmax_trading_rules] + return [trading_rules, ascend_ex_trading_rules] async def _update_account_data(self): headers = { - **self._bitmax_auth.get_headers(), - **self._bitmax_auth.get_auth_headers("info"), + **self._ascend_ex_auth.get_headers(), + **self._ascend_ex_auth.get_auth_headers("info"), } url = f"{REST_URL}/info" response = await aiohttp.ClientSession().get(url, headers=headers) @@ -359,16 +359,16 @@ async def _api_request(self, if (self._account_group) is None: await self._update_account_data() - url = f"{getRestUrlPriv(self._account_group)}/{path_url}" + url = f"{ascend_ex_utils.get_rest_url_private(self._account_group)}/{path_url}" headers = { - **self._bitmax_auth.get_headers(), - **self._bitmax_auth.get_auth_headers( + **self._ascend_ex_auth.get_headers(), + **self._ascend_ex_auth.get_auth_headers( path_url if force_auth_path_url is None else force_auth_path_url ), } else: url = f"{REST_URL}/{path_url}" - headers = self._bitmax_auth.get_headers() + headers = self._ascend_ex_auth.get_headers() client = await self._http_client() if method == "get": @@ -433,7 +433,7 @@ def buy(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, :param price: The price (note: this is no longer optional) :returns A new internal order id """ - client_order_id = bitmax_utils.gen_client_order_id(True, trading_pair) + client_order_id = ascend_ex_utils.gen_client_order_id(True, trading_pair) safe_ensure_future(self._create_order(TradeType.BUY, client_order_id, trading_pair, amount, order_type, price)) return client_order_id @@ -448,7 +448,7 @@ def sell(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, :param price: The price (note: this is no longer optional) :returns A new internal order id """ - client_order_id = bitmax_utils.gen_client_order_id(False, trading_pair) + client_order_id = ascend_ex_utils.gen_client_order_id(False, trading_pair) safe_ensure_future(self._create_order(TradeType.SELL, client_order_id, trading_pair, amount, order_type, price)) return client_order_id @@ -480,25 +480,25 @@ async def _create_order(self, """ if not order_type.is_limit_type(): raise Exception(f"Unsupported order type: {order_type}") - bitmax_trading_rule = self._bitmax_trading_rules[trading_pair] + ascend_ex_trading_rule = self._ascend_ex_trading_rules[trading_pair] amount = self.quantize_order_amount(trading_pair, amount) price = self.quantize_order_price(trading_pair, price) try: - # bitmax has a unique way of determening if the order has enough "worth" to be posted - # see https://bitmax-exchange.github.io/bitmax-pro-api/#place-order + # ascend_ex has a unique way of determening if the order has enough "worth" to be posted + # see https://ascendex.github.io/ascendex-pro-api/#place-order notional = Decimal(price * amount) - if notional < bitmax_trading_rule.minNotional or notional > bitmax_trading_rule.maxNotional: - raise ValueError(f"Notional amount {notional} is not withing the range of {bitmax_trading_rule.minNotional}-{bitmax_trading_rule.maxNotional}.") + if notional < ascend_ex_trading_rule.minNotional or notional > ascend_ex_trading_rule.maxNotional: + raise ValueError(f"Notional amount {notional} is not withing the range of {ascend_ex_trading_rule.minNotional}-{ascend_ex_trading_rule.maxNotional}.") # TODO: check balance - [exchange_order_id, timestamp] = bitmax_utils.gen_exchange_order_id(self._account_uid) + [exchange_order_id, timestamp] = ascend_ex_utils.gen_exchange_order_id(self._account_uid, order_id) api_params = { "id": exchange_order_id, "time": timestamp, - "symbol": bitmax_utils.convert_to_exchange_trading_pair(trading_pair), + "symbol": ascend_ex_utils.convert_to_exchange_trading_pair(trading_pair), "orderPrice": f"{price:f}", "orderQty": f"{amount:f}", "orderType": "limit", @@ -517,7 +517,6 @@ async def _create_order(self, await self._api_request("post", "cash/order", api_params, True, force_auth_path_url="order") tracked_order = self._in_flight_orders.get(order_id) - # tracked_order.update_exchange_order_id(exchange_order_id) if tracked_order is not None: self.logger().info(f"Created {order_type.name} {trade_type.name} order {order_id} for " @@ -540,7 +539,7 @@ async def _create_order(self, except Exception as e: self.stop_tracking_order(order_id) self.logger().network( - f"Error submitting {trade_type.name} {order_type.name} order to Bitmax for " + f"Error submitting {trade_type.name} {order_type.name} order to AscendEx for " f"{amount} {trading_pair} " f"{price}.", exc_info=True, @@ -560,7 +559,7 @@ def start_tracking_order(self, """ Starts tracking an order by simply adding it into _in_flight_orders dictionary. """ - self._in_flight_orders[order_id] = BitmaxInFlightOrder( + self._in_flight_orders[order_id] = AscendExInFlightOrder( client_order_id=order_id, exchange_order_id=exchange_order_id, trading_pair=trading_pair, @@ -596,9 +595,9 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: "delete", "cash/order", { - "symbol": bitmax_utils.convert_to_exchange_trading_pair(trading_pair), + "symbol": ascend_ex_utils.convert_to_exchange_trading_pair(trading_pair), "orderId": ex_order_id, - "time": bitmax_utils.get_ms_timestamp() + "time": ascend_ex_utils.get_ms_timestamp() }, True, force_auth_path_url="order" @@ -615,7 +614,7 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: self.logger().network( f"Failed to cancel order {order_id}: {str(e)}", exc_info=True, - app_warning_msg=f"Failed to cancel the order {order_id} on Bitmax. " + app_warning_msg=f"Failed to cancel the order {order_id} on AscendEx. " f"Check API key and network connection." ) @@ -639,7 +638,7 @@ async def _status_polling_loop(self): self.logger().error(str(e), exc_info=True) self.logger().network("Unexpected error while fetching account updates.", exc_info=True, - app_warning_msg="Could not fetch account updates from Bitmax. " + app_warning_msg="Could not fetch account updates from AscendEx. " "Check API key and network connection.") await asyncio.sleep(0.5) @@ -649,7 +648,7 @@ async def _update_balances(self): """ response = await self._api_request("get", "cash/balance", {}, True, force_auth_path_url="balance") balances = list(map( - lambda balance: BitmaxBalance( + lambda balance: AscendExBalance( balance["asset"], balance["availableBalance"], balance["totalBalance"] @@ -687,7 +686,7 @@ async def _update_order_status(self): continue order_data = response.get("data") - self._process_order_message(BitmaxOrder( + self._process_order_message(AscendExOrder( order_data["symbol"], order_data["price"], order_data["orderQty"], @@ -738,7 +737,7 @@ async def cancel_all(self, timeout_seconds: float): self.logger().network( "Failed to cancel all orders.", exc_info=True, - app_warning_msg="Failed to cancel all orders on Bitmax. Check API key and network connection." + app_warning_msg="Failed to cancel all orders on AscendEx. Check API key and network connection." ) return cancellation_results @@ -783,14 +782,14 @@ async def _iter_user_event_queue(self) -> AsyncIterable[Dict[str, any]]: self.logger().network( "Unknown error. Retrying after 1 seconds.", exc_info=True, - app_warning_msg="Could not fetch user events from Bitmax. Check API key and network connection." + app_warning_msg="Could not fetch user events from AscendEx. Check API key and network connection." ) await asyncio.sleep(1.0) async def _user_stream_event_listener(self): """ Listens to message in _user_stream_tracker.user_stream queue. The messages are put in by - BitmaxAPIUserStreamDataSource. + AscendExAPIUserStreamDataSource. """ async for event_message in self._iter_user_event_queue(): try: @@ -798,7 +797,7 @@ async def _user_stream_event_listener(self): order_data = event_message.get("data") trading_pair = order_data["s"] base_asset, quote_asset = tuple(asset for asset in trading_pair.split("/")) - self._process_order_message(BitmaxOrder( + self._process_order_message(AscendExOrder( trading_pair, order_data["p"], order_data["q"], @@ -817,12 +816,12 @@ async def _user_stream_event_listener(self): order_data["ei"] )) # Handles balance updates from orders. - base_asset_balance = BitmaxBalance( + base_asset_balance = AscendExBalance( base_asset, order_data["bab"], order_data["btb"] ) - quote_asset_balance = BitmaxBalance( + quote_asset_balance = AscendExBalance( quote_asset, order_data["qab"], order_data["qtb"] @@ -831,7 +830,7 @@ async def _user_stream_event_listener(self): elif event_message.get("m") == "balance": # Handles balance updates from Deposits/Withdrawals, Transfers between Cash and Margin Accounts balance_data = event_message.get("data") - balance = BitmaxBalance( + balance = AscendExBalance( balance_data["a"], balance_data["ab"], balance_data["tb"] @@ -869,7 +868,7 @@ async def get_open_orders(self) -> List[OpenOrder]: ret_val.append( OpenOrder( client_order_id=client_order_id, - trading_pair=bitmax_utils.convert_from_exchange_trading_pair(order["symbol"]), + trading_pair=ascend_ex_utils.convert_from_exchange_trading_pair(order["symbol"]), price=Decimal(str(order["price"])), amount=Decimal(str(order["orderQty"])), executed_amount=Decimal(str(order["cumFilledQty"])), @@ -882,7 +881,7 @@ async def get_open_orders(self) -> List[OpenOrder]: ) return ret_val - def _process_order_message(self, order_msg: BitmaxOrder): + def _process_order_message(self, order_msg: AscendExOrder): """ Updates in-flight order and triggers cancellation or failure event if needed. :param order_msg: The order response from either REST or web socket API (they are of the same format) @@ -917,7 +916,7 @@ def _process_order_message(self, order_msg: BitmaxOrder): self.stop_tracking_order(client_order_id) elif tracked_order.is_failure: self.logger().info(f"The market order {client_order_id} has failed according to order status API. " - f"Reason: {bitmax_utils.get_api_reason(order_msg.errorCode)}") + f"Reason: {ascend_ex_utils.get_api_reason(order_msg.errorCode)}") self.trigger_event(MarketEvent.OrderFailure, MarketOrderFailureEvent( self.current_timestamp, @@ -965,7 +964,7 @@ def _process_order_message(self, order_msg: BitmaxOrder): ) self.stop_tracking_order(client_order_id) - def _process_balances(self, balances: List[BitmaxBalance]): + def _process_balances(self, balances: List[AscendExBalance]): local_asset_names = set(self._account_balances.keys()) remote_asset_names = set() diff --git a/hummingbot/connector/exchange/bitmax/bitmax_in_flight_order.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_in_flight_order.py similarity index 95% rename from hummingbot/connector/exchange/bitmax/bitmax_in_flight_order.py rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_in_flight_order.py index b455433ba0..0654f2cba9 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_in_flight_order.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_in_flight_order.py @@ -12,7 +12,7 @@ from hummingbot.connector.in_flight_order_base import InFlightOrderBase -class BitmaxInFlightOrder(InFlightOrderBase): +class AscendExInFlightOrder(InFlightOrderBase): def __init__(self, client_order_id: str, exchange_order_id: Optional[str], @@ -53,7 +53,7 @@ def from_json(cls, data: Dict[str, Any]) -> InFlightOrderBase: :param data: json data from API :return: formatted InFlightOrder """ - retval = BitmaxInFlightOrder( + retval = AscendExInFlightOrder( data["client_order_id"], data["exchange_order_id"], data["trading_pair"], diff --git a/hummingbot/connector/exchange/bitmax/bitmax_order_book.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book.py similarity index 83% rename from hummingbot/connector/exchange/bitmax/bitmax_order_book.py rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book.py index be6702853d..88ebe8b0d1 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_order_book.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book.py @@ -1,24 +1,25 @@ #!/usr/bin/env python import logging -import hummingbot.connector.exchange.bitmax.bitmax_constants as constants +import hummingbot.connector.exchange.ascend_ex.ascend_ex_constants as constants from sqlalchemy.engine import RowProxy from typing import ( Optional, Dict, List, Any) -from hummingbot.logger import HummingbotLogger + from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.order_book_message import ( OrderBookMessage, OrderBookMessageType ) -from hummingbot.connector.exchange.bitmax.bitmax_order_book_message import BitmaxOrderBookMessage +from hummingbot.connector.exchange.ascend_ex.ascend_ex_order_book_message import AscendExOrderBookMessage +from hummingbot.logger import HummingbotLogger _logger = None -class BitmaxOrderBook(OrderBook): +class AscendExOrderBook(OrderBook): @classmethod def logger(cls) -> HummingbotLogger: global _logger @@ -35,13 +36,13 @@ def snapshot_message_from_exchange(cls, Convert json snapshot data into standard OrderBookMessage format :param msg: json snapshot data from live web socket stream :param timestamp: timestamp attached to incoming data - :return: BitmaxOrderBookMessage + :return: AscendExOrderBookMessage """ if metadata: msg.update(metadata) - return BitmaxOrderBookMessage( + return AscendExOrderBookMessage( message_type=OrderBookMessageType.SNAPSHOT, content=msg, timestamp=timestamp @@ -53,9 +54,9 @@ def snapshot_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = N *used for backtesting Convert a row of snapshot data into standard OrderBookMessage format :param record: a row of snapshot data from the database - :return: BitmaxOrderBookMessage + :return: AscendExOrderBookMessage """ - return BitmaxOrderBookMessage( + return AscendExOrderBookMessage( message_type=OrderBookMessageType.SNAPSHOT, content=record.json, timestamp=record.timestamp @@ -70,13 +71,13 @@ def diff_message_from_exchange(cls, Convert json diff data into standard OrderBookMessage format :param msg: json diff data from live web socket stream :param timestamp: timestamp attached to incoming data - :return: BitmaxOrderBookMessage + :return: AscendExOrderBookMessage """ if metadata: msg.update(metadata) - return BitmaxOrderBookMessage( + return AscendExOrderBookMessage( message_type=OrderBookMessageType.DIFF, content=msg, timestamp=timestamp @@ -88,9 +89,9 @@ def diff_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None) *used for backtesting Convert a row of diff data into standard OrderBookMessage format :param record: a row of diff data from the database - :return: BitmaxOrderBookMessage + :return: AscendExOrderBookMessage """ - return BitmaxOrderBookMessage( + return AscendExOrderBookMessage( message_type=OrderBookMessageType.DIFF, content=record.json, timestamp=record.timestamp @@ -104,7 +105,7 @@ def trade_message_from_exchange(cls, """ Convert a trade data into standard OrderBookMessage format :param record: a trade data from the database - :return: BitmaxOrderBookMessage + :return: AscendExOrderBookMessage """ if metadata: @@ -117,7 +118,7 @@ def trade_message_from_exchange(cls, "amount": msg.get("q"), }) - return BitmaxOrderBookMessage( + return AscendExOrderBookMessage( message_type=OrderBookMessageType.TRADE, content=msg, timestamp=timestamp @@ -129,9 +130,9 @@ def trade_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None *used for backtesting Convert a row of trade data into standard OrderBookMessage format :param record: a row of trade data from the database - :return: BitmaxOrderBookMessage + :return: AscendExOrderBookMessage """ - return BitmaxOrderBookMessage( + return AscendExOrderBookMessage( message_type=OrderBookMessageType.TRADE, content=record.json, timestamp=record.timestamp @@ -142,5 +143,5 @@ def from_snapshot(cls, snapshot: OrderBookMessage): raise NotImplementedError(constants.EXCHANGE_NAME + " order book needs to retain individual order data.") @classmethod - def restore_from_snapshot_and_diffs(self, snapshot: OrderBookMessage, diffs: List[OrderBookMessage]): + def restore_from_snapshot_and_diffs(cls, snapshot: OrderBookMessage, diffs: List[OrderBookMessage]): raise NotImplementedError(constants.EXCHANGE_NAME + " order book needs to retain individual order data.") diff --git a/hummingbot/connector/exchange/bitmax/bitmax_order_book_message.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book_message.py similarity index 95% rename from hummingbot/connector/exchange/bitmax/bitmax_order_book_message.py rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book_message.py index a00550b66b..c5b0ef2e13 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_order_book_message.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book_message.py @@ -13,7 +13,7 @@ ) -class BitmaxOrderBookMessage(OrderBookMessage): +class AscendExOrderBookMessage(OrderBookMessage): def __new__( cls, message_type: OrderBookMessageType, @@ -27,7 +27,7 @@ def __new__( raise ValueError("timestamp must not be None when initializing snapshot messages.") timestamp = content["timestamp"] - return super(BitmaxOrderBookMessage, cls).__new__( + return super(AscendExOrderBookMessage, cls).__new__( cls, message_type, content, timestamp=timestamp, *args, **kwargs ) diff --git a/hummingbot/connector/exchange/bitmax/bitmax_order_book_tracker.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book_tracker.py similarity index 75% rename from hummingbot/connector/exchange/bitmax/bitmax_order_book_tracker.py rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book_tracker.py index 9fa26cc508..0801dd47be 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_order_book_tracker.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book_tracker.py @@ -2,20 +2,23 @@ import asyncio import bisect import logging -import hummingbot.connector.exchange.bitmax.bitmax_constants as constants import time + +import hummingbot.connector.exchange.ascend_ex.ascend_ex_constants as constants + from collections import defaultdict, deque from typing import Optional, Dict, List, Deque + from hummingbot.core.data_type.order_book_message import OrderBookMessageType -from hummingbot.logger import HummingbotLogger from hummingbot.core.data_type.order_book_tracker import OrderBookTracker -from hummingbot.connector.exchange.bitmax.bitmax_order_book_message import BitmaxOrderBookMessage -from hummingbot.connector.exchange.bitmax.bitmax_active_order_tracker import BitmaxActiveOrderTracker -from hummingbot.connector.exchange.bitmax.bitmax_api_order_book_data_source import BitmaxAPIOrderBookDataSource -from hummingbot.connector.exchange.bitmax.bitmax_order_book import BitmaxOrderBook +from hummingbot.connector.exchange.ascend_ex.ascend_ex_order_book_message import AscendExOrderBookMessage +from hummingbot.connector.exchange.ascend_ex.ascend_ex_active_order_tracker import AscendExActiveOrderTracker +from hummingbot.connector.exchange.ascend_ex.ascend_ex_api_order_book_data_source import AscendExAPIOrderBookDataSource +from hummingbot.connector.exchange.ascend_ex.ascend_ex_order_book import AscendExOrderBook +from hummingbot.logger import HummingbotLogger -class BitmaxOrderBookTracker(OrderBookTracker): +class AscendExOrderBookTracker(OrderBookTracker): _logger: Optional[HummingbotLogger] = None @classmethod @@ -25,7 +28,7 @@ def logger(cls) -> HummingbotLogger: return cls._logger def __init__(self, trading_pairs: Optional[List[str]] = None,): - super().__init__(BitmaxAPIOrderBookDataSource(trading_pairs), trading_pairs) + super().__init__(AscendExAPIOrderBookDataSource(trading_pairs), trading_pairs) self._ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() self._order_book_snapshot_stream: asyncio.Queue = asyncio.Queue() @@ -33,10 +36,10 @@ def __init__(self, trading_pairs: Optional[List[str]] = None,): self._order_book_trade_stream: asyncio.Queue = asyncio.Queue() self._process_msg_deque_task: Optional[asyncio.Task] = None self._past_diffs_windows: Dict[str, Deque] = {} - self._order_books: Dict[str, BitmaxOrderBook] = {} - self._saved_message_queues: Dict[str, Deque[BitmaxOrderBookMessage]] = \ + self._order_books: Dict[str, AscendExOrderBook] = {} + self._saved_message_queues: Dict[str, Deque[AscendExOrderBookMessage]] = \ defaultdict(lambda: deque(maxlen=1000)) - self._active_order_trackers: Dict[str, BitmaxActiveOrderTracker] = defaultdict(BitmaxActiveOrderTracker) + self._active_order_trackers: Dict[str, AscendExActiveOrderTracker] = defaultdict(AscendExActiveOrderTracker) self._order_book_stream_listener_task: Optional[asyncio.Task] = None self._order_book_trade_listener_task: Optional[asyncio.Task] = None @@ -51,20 +54,20 @@ async def _track_single_book(self, trading_pair: str): """ Update an order book with changes from the latest batch of received messages """ - past_diffs_window: Deque[BitmaxOrderBookMessage] = deque() + past_diffs_window: Deque[AscendExOrderBookMessage] = deque() self._past_diffs_windows[trading_pair] = past_diffs_window message_queue: asyncio.Queue = self._tracking_message_queues[trading_pair] - order_book: BitmaxOrderBook = self._order_books[trading_pair] - active_order_tracker: BitmaxActiveOrderTracker = self._active_order_trackers[trading_pair] + order_book: AscendExOrderBook = self._order_books[trading_pair] + active_order_tracker: AscendExActiveOrderTracker = self._active_order_trackers[trading_pair] last_message_timestamp: float = time.time() diff_messages_accepted: int = 0 while True: try: - message: BitmaxOrderBookMessage = None - saved_messages: Deque[BitmaxOrderBookMessage] = self._saved_message_queues[trading_pair] + message: AscendExOrderBookMessage = None + saved_messages: Deque[AscendExOrderBookMessage] = self._saved_message_queues[trading_pair] # Process saved messages first if there are any if len(saved_messages) > 0: message = saved_messages.popleft() @@ -87,7 +90,7 @@ async def _track_single_book(self, trading_pair: str): diff_messages_accepted = 0 last_message_timestamp = now elif message.type is OrderBookMessageType.SNAPSHOT: - past_diffs: List[BitmaxOrderBookMessage] = list(past_diffs_window) + past_diffs: List[AscendExOrderBookMessage] = list(past_diffs_window) # only replay diffs later than snapshot, first update active order with snapshot then replay diffs replay_position = bisect.bisect_right(past_diffs, message) replay_diffs = past_diffs[replay_position:] diff --git a/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book_tracker_entry.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book_tracker_entry.py new file mode 100644 index 0000000000..ebefecfebb --- /dev/null +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_order_book_tracker_entry.py @@ -0,0 +1,21 @@ +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_tracker_entry import OrderBookTrackerEntry +from hummingbot.connector.exchange.ascend_ex.ascend_ex_active_order_tracker import AscendExActiveOrderTracker + + +class AscendExOrderBookTrackerEntry(OrderBookTrackerEntry): + def __init__( + self, trading_pair: str, timestamp: float, order_book: OrderBook, active_order_tracker: AscendExActiveOrderTracker + ): + self._active_order_tracker = active_order_tracker + super(AscendExOrderBookTrackerEntry, self).__init__(trading_pair, timestamp, order_book) + + def __repr__(self) -> str: + return ( + f"AscendExOrderBookTrackerEntry(trading_pair='{self._trading_pair}', timestamp='{self._timestamp}', " + f"order_book='{self._order_book}')" + ) + + @property + def active_order_tracker(self) -> AscendExActiveOrderTracker: + return self._active_order_tracker diff --git a/hummingbot/connector/exchange/ascend_ex/ascend_ex_user_stream_tracker.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_user_stream_tracker.py new file mode 100644 index 0000000000..8557ca44db --- /dev/null +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_user_stream_tracker.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python + +import asyncio +import logging + +from typing import ( + Optional, + List, +) +from hummingbot.connector.exchange.ascend_ex.ascend_ex_api_user_stream_data_source import \ + AscendExAPIUserStreamDataSource +from hummingbot.connector.exchange.ascend_ex.ascend_ex_auth import AscendExAuth +from hummingbot.connector.exchange.ascend_ex.ascend_ex_constants import EXCHANGE_NAME +from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource +from hummingbot.core.data_type.user_stream_tracker import ( + UserStreamTracker +) +from hummingbot.core.utils.async_utils import ( + safe_ensure_future, + safe_gather, +) + +from hummingbot.logger import HummingbotLogger + + +class AscendExUserStreamTracker(UserStreamTracker): + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, + ascend_ex_auth: Optional[AscendExAuth] = None, + trading_pairs: Optional[List[str]] = []): + super().__init__() + self._ascend_ex_auth: AscendExAuth = ascend_ex_auth + self._trading_pairs: List[str] = trading_pairs + self._ev_loop: asyncio.events.AbstractEventLoop = asyncio.get_event_loop() + self._data_source: Optional[UserStreamTrackerDataSource] = None + self._user_stream_tracking_task: Optional[asyncio.Task] = None + + @property + def data_source(self) -> UserStreamTrackerDataSource: + """ + *required + Initializes a user stream data source (user specific order diffs from live socket stream) + :return: OrderBookTrackerDataSource + """ + if not self._data_source: + self._data_source = AscendExAPIUserStreamDataSource( + ascend_ex_auth=self._ascend_ex_auth, + trading_pairs=self._trading_pairs + ) + return self._data_source + + @property + def exchange_name(self) -> str: + """ + *required + Name of the current exchange + """ + return EXCHANGE_NAME + + async def start(self): + """ + *required + Start all listeners and tasks + """ + self._user_stream_tracking_task = safe_ensure_future( + self.data_source.listen_for_user_stream(self._ev_loop, self._user_stream) + ) + await safe_gather(self._user_stream_tracking_task) diff --git a/hummingbot/connector/exchange/bitmax/bitmax_utils.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_utils.py similarity index 57% rename from hummingbot/connector/exchange/bitmax/bitmax_utils.py rename to hummingbot/connector/exchange/ascend_ex/ascend_ex_utils.py index 45f943688d..a8ac64400f 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_utils.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_utils.py @@ -15,7 +15,15 @@ DEFAULT_FEES = [0.1, 0.1] -HBOT_BROKER_ID = "hbot-" +HBOT_BROKER_ID = "HMBot" + + +def get_rest_url_private(account_id: int) -> str: + return f"https://ascendex.com/{account_id}/api/pro/v1" + + +def get_ws_url_private(account_id: int) -> str: + return f"wss://ascendex.com/{account_id}/api/pro/v1" def convert_from_exchange_trading_pair(exchange_trading_pair: str) -> str: @@ -35,29 +43,29 @@ def uuid32(): return ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(32)) -def derive_order_id(user_uid: str, cl_order_id: str, ts: int, order_src='a') -> str: +def derive_order_id(user_uid: str, cl_order_id: str, ts: int) -> str: """ Server order generator based on user info and input. :param user_uid: user uid :param cl_order_id: user random digital and number id :param ts: order timestamp in milliseconds - :param order_src: 'a' for rest api order, 's' for websocket order. :return: order id of length 32 """ - return (order_src + format(ts, 'x')[-11:] + user_uid[-11:] + cl_order_id[-9:])[:32] + return (HBOT_BROKER_ID + format(ts, 'x')[-11:] + user_uid[-11:] + cl_order_id[-5:])[:32] -def gen_exchange_order_id(userUid: str) -> Tuple[str, int]: +def gen_exchange_order_id(userUid: str, client_order_id: str) -> Tuple[str, int]: """ - Generate an order id - :param user_uid: user uid + Generates the exchange order id based on user uid and client order id. + :param user_uid: user uid, + :param client_order_id: client order id used for local order tracking :return: order id of length 32 """ time = get_ms_timestamp() return [ derive_order_id( userUid, - uuid32(), + client_order_id, time ), time @@ -66,20 +74,20 @@ def gen_exchange_order_id(userUid: str) -> Tuple[str, int]: def gen_client_order_id(is_buy: bool, trading_pair: str) -> str: side = "B" if is_buy else "S" - return f"{HBOT_BROKER_ID}{side}-{trading_pair}-{get_tracking_nonce()}" + return f"{HBOT_BROKER_ID}-{side}-{trading_pair}-{get_tracking_nonce()}" KEYS = { - "bitmax_api_key": - ConfigVar(key="bitmax_api_key", - prompt="Enter your Bitmax API key >>> ", - required_if=using_exchange("bitmax"), + "ascend_ex_api_key": + ConfigVar(key="ascend_ex_api_key", + prompt="Enter your AscendEx API key >>> ", + required_if=using_exchange("ascend_ex"), is_secure=True, is_connect_key=True), - "bitmax_secret_key": - ConfigVar(key="bitmax_secret_key", - prompt="Enter your Bitmax secret key >>> ", - required_if=using_exchange("bitmax"), + "ascend_ex_secret_key": + ConfigVar(key="ascend_ex_secret_key", + prompt="Enter your AscendEx secret key >>> ", + required_if=using_exchange("ascend_ex"), is_secure=True, is_connect_key=True), } diff --git a/hummingbot/connector/exchange/beaxy/beaxy_api_order_book_data_source.py b/hummingbot/connector/exchange/beaxy/beaxy_api_order_book_data_source.py index f42881c33f..5be1e9257a 100644 --- a/hummingbot/connector/exchange/beaxy/beaxy_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/beaxy/beaxy_api_order_book_data_source.py @@ -5,13 +5,10 @@ import asyncio import json import ujson -import cachetools.func from typing import Any, AsyncIterable, Optional, List, Dict -from decimal import Decimal - import pandas as pd import websockets -import requests + from websockets.exceptions import ConnectionClosed from hummingbot.logger import HummingbotLogger @@ -112,16 +109,6 @@ async def fetch_trading_pairs() -> Optional[List[str]]: pass return [] - @staticmethod - @cachetools.func.ttl_cache(ttl=10) - def get_mid_price(trading_pair: str) -> Optional[Decimal]: - - symbol = trading_pair_to_symbol(trading_pair) - resp = requests.get(url=BeaxyConstants.PublicApi.RATE_URL.format(symbol=symbol)) - record = resp.json() - - return (Decimal(record['bid']) + Decimal(record['ask'])) / Decimal('2') - @classmethod async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, float]: diff --git a/hummingbot/connector/exchange/binance/binance_api_order_book_data_source.py b/hummingbot/connector/exchange/binance/binance_api_order_book_data_source.py index 253f5eb4b3..7215fc29d6 100755 --- a/hummingbot/connector/exchange/binance/binance_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/binance/binance_api_order_book_data_source.py @@ -13,8 +13,6 @@ ) from decimal import Decimal import re -import requests -import cachetools.func import time import ujson import websockets @@ -68,16 +66,6 @@ async def get_last_traded_price(cls, trading_pair: str, domain: str = "com") -> resp_json = await resp.json() return float(resp_json["lastPrice"]) - @staticmethod - @cachetools.func.ttl_cache(ttl=10, maxsize=1000) - def get_mid_price(trading_pair: str, domain="com") -> Optional[Decimal]: - from hummingbot.connector.exchange.binance.binance_utils import convert_to_exchange_trading_pair - url = TICKER_PRICE_CHANGE_URL.format(domain) - resp = requests.get(url=f"{url}?symbol={convert_to_exchange_trading_pair(trading_pair)}") - record = resp.json() - result = (Decimal(record.get("bidPrice", "0")) + Decimal(record.get("askPrice", "0"))) / Decimal("2") - return result if result else None - @staticmethod @async_ttl_cache(ttl=2, maxsize=1) async def get_all_mid_prices(domain="com") -> Optional[Decimal]: diff --git a/hummingbot/connector/exchange/bitfinex/bitfinex_api_order_book_data_source.py b/hummingbot/connector/exchange/bitfinex/bitfinex_api_order_book_data_source.py index 75754b942a..aee07068de 100644 --- a/hummingbot/connector/exchange/bitfinex/bitfinex_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/bitfinex/bitfinex_api_order_book_data_source.py @@ -6,9 +6,6 @@ import asyncio import ujson import pandas as pd -from decimal import Decimal -import requests -import cachetools.func from typing import ( Any, AsyncIterable, @@ -91,15 +88,6 @@ def __init__(self, trading_pairs: Optional[List[str]] = None): # way it is stored in Hummingbot order book, usually timestamp) self._tracked_book_entries: Dict[int, OrderBookRow] = {} - @staticmethod - @cachetools.func.ttl_cache(ttl=10) - def get_mid_price(trading_pair: str) -> Optional[Decimal]: - exchange_trading_pair = convert_to_exchange_trading_pair(trading_pair) - resp = requests.get(url=f"https://api-pub.bitfinex.com/v2/ticker/{exchange_trading_pair}") - record = resp.json() - result = (Decimal(record[0]) + Decimal(record[2])) / Decimal("2") - return result - @staticmethod async def fetch_trading_pairs() -> List[str]: try: diff --git a/hummingbot/connector/exchange/bitmax/bitmax_constants.py b/hummingbot/connector/exchange/bitmax/bitmax_constants.py deleted file mode 100644 index 51ec061543..0000000000 --- a/hummingbot/connector/exchange/bitmax/bitmax_constants.py +++ /dev/null @@ -1,15 +0,0 @@ -# A single source of truth for constant variables related to the exchange - - -EXCHANGE_NAME = "bitmax" -REST_URL = "https://bitmax.io/api/pro/v1" -WS_URL = "wss://bitmax.io/1/api/pro/v1/stream" -PONG_PAYLOAD = {"op": "pong"} - - -def getRestUrlPriv(accountId: int) -> str: - return f"https://bitmax.io/{accountId}/api/pro/v1" - - -def getWsUrlPriv(accountId: int) -> str: - return f"wss://bitmax.io/{accountId}/api/pro/v1" diff --git a/hummingbot/connector/exchange/bittrex/bittrex_api_order_book_data_source.py b/hummingbot/connector/exchange/bittrex/bittrex_api_order_book_data_source.py index 1d125ea85e..b379106836 100644 --- a/hummingbot/connector/exchange/bittrex/bittrex_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/bittrex/bittrex_api_order_book_data_source.py @@ -14,10 +14,6 @@ from signalr_aio.hubs import Hub from async_timeout import timeout -import requests -import cachetools.func -from decimal import Decimal - from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.order_book_message import OrderBookMessage from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource @@ -101,20 +97,6 @@ async def websocket_connection(self) -> (signalr_aio.Connection, signalr_aio.hub return self._websocket_connection, self._websocket_hub - @staticmethod - @cachetools.func.ttl_cache(ttl=10) - def get_mid_price(trading_pair: str) -> Optional[Decimal]: - resp = requests.get(url="https://api.bittrex.com/api/v1.1/public/getmarketsummaries") - records = resp.json() - result = None - for record in records["result"]: - symbols = record["MarketName"].split("-") - pair = f"{symbols[1]}-{symbols[0]}" - if trading_pair == pair and record["Bid"] is not None and record["Ask"] is not None: - result = (Decimal(record["Bid"]) + Decimal(record["Ask"])) / Decimal("2") - break - return result - @staticmethod async def fetch_trading_pairs() -> List[str]: try: diff --git a/hummingbot/connector/exchange/blocktane/blocktane_api_order_book_data_source.py b/hummingbot/connector/exchange/blocktane/blocktane_api_order_book_data_source.py index 71198bb609..cdf4d55c32 100644 --- a/hummingbot/connector/exchange/blocktane/blocktane_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/blocktane/blocktane_api_order_book_data_source.py @@ -2,9 +2,7 @@ import asyncio import aiohttp -import cachetools.func from collections import namedtuple -from decimal import Decimal import logging import pandas as pd from typing import ( @@ -15,7 +13,6 @@ Optional ) import re -import requests import time import ujson import websockets @@ -68,20 +65,6 @@ async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, flo def trading_pairs(self) -> List[str]: return self._trading_pairs - @staticmethod - @cachetools.func.ttl_cache(ttl=10) - def get_mid_price(trading_pair: str) -> Optional[Decimal]: - exchange_trading_pair: str = convert_to_exchange_trading_pair(trading_pair) - - try: - resp = requests.get(url=f"{SINGLE_MARKET_DEPTH_URL.format(exchange_trading_pair)}?limit=1").json() - best_bid: Decimal = Decimal(resp["bids"][0][0]) - best_ask: Decimal = Decimal(resp["ask"][0][0]) - - return (best_ask + best_bid) / 2 - except Exception: - return None - @staticmethod async def fetch_trading_pairs() -> List[str]: try: diff --git a/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_api_order_book_data_source.py b/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_api_order_book_data_source.py index 95bca8b062..0b97fd45ce 100755 --- a/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_api_order_book_data_source.py @@ -11,15 +11,10 @@ List, Optional, ) -from decimal import Decimal import time import ujson import websockets from websockets.exceptions import ConnectionClosed - -import requests -import cachetools.func - from hummingbot.core.data_type.order_book import OrderBook from hummingbot.connector.exchange.coinbase_pro.coinbase_pro_order_book import CoinbaseProOrderBook from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource @@ -66,16 +61,6 @@ async def get_last_traded_price(cls, trading_pair: str) -> float: resp_json = await resp.json() return float(resp_json["price"]) - @staticmethod - @cachetools.func.ttl_cache(ttl=10) - def get_mid_price(trading_pair: str) -> Optional[Decimal]: - COINBASE_PRO_PRICE_URL = "https://api.pro.coinbase.com/products/TO_BE_REPLACED/ticker" - resp = requests.get(url=COINBASE_PRO_PRICE_URL.replace("TO_BE_REPLACED", trading_pair)) - record = resp.json() - if "bid" in record and "ask" in record: - result = (Decimal(record["bid"]) + Decimal(record["ask"])) / Decimal("2") - return result - @staticmethod async def fetch_trading_pairs() -> List[str]: try: diff --git a/test/connector/exchange/bitmax/__init__.py b/hummingbot/connector/exchange/digifinex/__init__.py similarity index 100% rename from test/connector/exchange/bitmax/__init__.py rename to hummingbot/connector/exchange/digifinex/__init__.py diff --git a/hummingbot/connector/exchange/digifinex/digifinex_active_order_tracker.pxd b/hummingbot/connector/exchange/digifinex/digifinex_active_order_tracker.pxd new file mode 100644 index 0000000000..f9fdfcd81e --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_active_order_tracker.pxd @@ -0,0 +1,10 @@ +# distutils: language=c++ +cimport numpy as np + +cdef class DigifinexActiveOrderTracker: + cdef dict _active_bids + cdef dict _active_asks + + cdef tuple c_convert_diff_message_to_np_arrays(self, object message) + cdef tuple c_convert_snapshot_message_to_np_arrays(self, object message) + cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message) diff --git a/hummingbot/connector/exchange/digifinex/digifinex_active_order_tracker.pyx b/hummingbot/connector/exchange/digifinex/digifinex_active_order_tracker.pyx new file mode 100644 index 0000000000..19ff274da7 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_active_order_tracker.pyx @@ -0,0 +1,169 @@ +# distutils: language=c++ +# distutils: sources=hummingbot/core/cpp/OrderBookEntry.cpp + +import logging +import numpy as np + +from decimal import Decimal +from typing import Dict +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.order_book_row import OrderBookRow + +_logger = None +s_empty_diff = np.ndarray(shape=(0, 4), dtype="float64") +DigifinexOrderBookTrackingDictionary = Dict[Decimal, Dict[str, Dict[str, any]]] + +cdef class DigifinexActiveOrderTracker: + def __init__(self, + active_asks: DigifinexOrderBookTrackingDictionary = None, + active_bids: DigifinexOrderBookTrackingDictionary = None): + super().__init__() + self._active_asks = active_asks or {} + self._active_bids = active_bids or {} + + @classmethod + def logger(cls) -> HummingbotLogger: + global _logger + if _logger is None: + _logger = logging.getLogger(__name__) + return _logger + + @property + def active_asks(self) -> DigifinexOrderBookTrackingDictionary: + return self._active_asks + + @property + def active_bids(self) -> DigifinexOrderBookTrackingDictionary: + return self._active_bids + + # TODO: research this more + def volume_for_ask_price(self, price) -> float: + return NotImplementedError + + # TODO: research this more + def volume_for_bid_price(self, price) -> float: + return NotImplementedError + + def get_rates_and_quantities(self, entry) -> tuple: + # price, quantity + return float(entry[0]), float(entry[1]) + + cdef tuple c_convert_diff_message_to_np_arrays(self, object message): + cdef: + dict content = message.content + list bid_entries = [] + list ask_entries = [] + str order_id + str order_side + str price_raw + object price + dict order_dict + double timestamp = message.timestamp + double amount = 0 + + bid_entries = content["bids"] + ask_entries = content["asks"] + + bids = s_empty_diff + asks = s_empty_diff + + if len(bid_entries) > 0: + bids = np.array( + [[timestamp, + float(price), + float(amount), + message.update_id] + for price, amount in [self.get_rates_and_quantities(entry) for entry in bid_entries]], + dtype="float64", + ndmin=2 + ) + + if len(ask_entries) > 0: + asks = np.array( + [[timestamp, + float(price), + float(amount), + message.update_id] + for price, amount in [self.get_rates_and_quantities(entry) for entry in ask_entries]], + dtype="float64", + ndmin=2 + ) + + return bids, asks + + cdef tuple c_convert_snapshot_message_to_np_arrays(self, object message): + cdef: + float price + float amount + str order_id + dict order_dict + + # Refresh all order tracking. + self._active_bids.clear() + self._active_asks.clear() + timestamp = message.timestamp + content = message.content + + for snapshot_orders, active_orders in [(content["bids"], self._active_bids), (content["asks"], self.active_asks)]: + for order in snapshot_orders: + price, amount = self.get_rates_and_quantities(order) + + order_dict = { + "order_id": timestamp, + "amount": amount + } + + if price in active_orders: + active_orders[price][timestamp] = order_dict + else: + active_orders[price] = { + timestamp: order_dict + } + + cdef: + np.ndarray[np.float64_t, ndim=2] bids = np.array( + [[message.timestamp, + price, + sum([order_dict["amount"] + for order_dict in self._active_bids[price].values()]), + message.update_id] + for price in sorted(self._active_bids.keys(), reverse=True)], dtype="float64", ndmin=2) + np.ndarray[np.float64_t, ndim=2] asks = np.array( + [[message.timestamp, + price, + sum([order_dict["amount"] + for order_dict in self.active_asks[price].values()]), + message.update_id] + for price in sorted(self.active_asks.keys(), reverse=True)], dtype="float64", ndmin=2 + ) + + if bids.shape[1] != 4: + bids = bids.reshape((0, 4)) + if asks.shape[1] != 4: + asks = asks.reshape((0, 4)) + + return bids, asks + + cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message): + cdef: + double trade_type_value = 2.0 + + timestamp = message.timestamp + content = message.content + + return np.array( + [timestamp, trade_type_value, float(content["price"]), float(content["size"])], + dtype="float64" + ) + + def convert_diff_message_to_order_book_row(self, message): + np_bids, np_asks = self.c_convert_diff_message_to_np_arrays(message) + bids_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_bids] + asks_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_asks] + return bids_row, asks_row + + def convert_snapshot_message_to_order_book_row(self, message): + np_bids, np_asks = self.c_convert_snapshot_message_to_np_arrays(message) + bids_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_bids] + asks_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_asks] + return bids_row, asks_row diff --git a/hummingbot/connector/exchange/digifinex/digifinex_api_order_book_data_source.py b/hummingbot/connector/exchange/digifinex/digifinex_api_order_book_data_source.py new file mode 100644 index 0000000000..b4162b365b --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_api_order_book_data_source.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python +import asyncio +import logging +import time +import aiohttp +import traceback +import pandas as pd +import hummingbot.connector.exchange.digifinex.digifinex_constants as constants + +from typing import Optional, List, Dict, Any +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_message import OrderBookMessage +from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource +from hummingbot.core.utils.async_utils import safe_gather +from hummingbot.logger import HummingbotLogger +from . import digifinex_utils +from .digifinex_active_order_tracker import DigifinexActiveOrderTracker +from .digifinex_order_book import DigifinexOrderBook +from .digifinex_websocket import DigifinexWebsocket +# from .digifinex_utils import ms_timestamp_to_s + + +class DigifinexAPIOrderBookDataSource(OrderBookTrackerDataSource): + MAX_RETRIES = 20 + MESSAGE_TIMEOUT = 30.0 + SNAPSHOT_TIMEOUT = 10.0 + + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, trading_pairs: List[str] = None): + super().__init__(trading_pairs) + self._trading_pairs: List[str] = trading_pairs + self._snapshot_msg: Dict[str, any] = {} + + @classmethod + async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, float]: + result = {} + async with aiohttp.ClientSession() as client: + resp = await client.get(f"{constants.REST_URL}/ticker") + resp_json = await resp.json() + for t_pair in trading_pairs: + last_trade = [o["last"] for o in resp_json["ticker"] if o["symbol"] == + digifinex_utils.convert_to_exchange_trading_pair(t_pair)] + if last_trade and last_trade[0] is not None: + result[t_pair] = last_trade[0] + return result + + @staticmethod + async def fetch_trading_pairs() -> List[str]: + async with aiohttp.ClientSession() as client: + async with client.get(f"{constants.REST_URL}/ticker", timeout=10) as response: + if response.status == 200: + from hummingbot.connector.exchange.digifinex.digifinex_utils import \ + convert_from_exchange_trading_pair + try: + data: Dict[str, Any] = await response.json() + return [convert_from_exchange_trading_pair(item["symbol"]) for item in data["ticker"]] + except Exception: + pass + # Do nothing if the request fails -- there will be no autocomplete for kucoin trading pairs + return [] + + @staticmethod + async def get_order_book_data(trading_pair: str) -> Dict[str, any]: + """ + Get whole orderbook + """ + async with aiohttp.ClientSession() as client: + orderbook_response = await client.get( + f"{constants.REST_URL}/order_book?limit=150&symbol=" + f"{digifinex_utils.convert_to_exchange_trading_pair(trading_pair)}" + ) + + if orderbook_response.status != 200: + raise IOError( + f"Error fetching OrderBook for {trading_pair} at {constants.EXCHANGE_NAME}. " + f"HTTP status is {orderbook_response.status}." + ) + + orderbook_data: List[Dict[str, Any]] = await safe_gather(orderbook_response.json()) + orderbook_data = orderbook_data[0] + return orderbook_data + + async def get_new_order_book(self, trading_pair: str) -> OrderBook: + snapshot: Dict[str, Any] = await self.get_order_book_data(trading_pair) + snapshot_timestamp: float = time.time() + snapshot_msg: OrderBookMessage = DigifinexOrderBook.snapshot_message_from_exchange( + snapshot, + snapshot_timestamp, + metadata={"trading_pair": trading_pair} + ) + order_book = self.order_book_create_function() + active_order_tracker: DigifinexActiveOrderTracker = DigifinexActiveOrderTracker() + bids, asks = active_order_tracker.convert_snapshot_message_to_order_book_row(snapshot_msg) + order_book.apply_snapshot(bids, asks, snapshot_msg.update_id) + return order_book + + async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for trades using websocket trade channel + """ + while True: + try: + ws = DigifinexWebsocket() + await ws.connect() + + await ws.subscribe("trades", list(map( + lambda pair: f"{digifinex_utils.convert_to_ws_trading_pair(pair)}", + self._trading_pairs + ))) + + async for response in ws.on_message(): + params = response["params"] + symbol = params[2] + for trade in params[1]: + trade_timestamp: int = trade["time"] + trade_msg: OrderBookMessage = DigifinexOrderBook.trade_message_from_exchange( + trade, + trade_timestamp, + metadata={"trading_pair": digifinex_utils.convert_from_ws_trading_pair(symbol)} + ) + output.put_nowait(trade_msg) + + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error.", exc_info=True) + await asyncio.sleep(5.0) + finally: + await ws.disconnect() + + async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for orderbook diffs using websocket book channel + """ + while True: + try: + ws = DigifinexWebsocket() + await ws.connect() + + await ws.subscribe("depth", list(map( + lambda pair: f"{digifinex_utils.convert_to_ws_trading_pair(pair)}", + self._trading_pairs + ))) + + async for response in ws.on_message(): + if response is None or 'params' not in response: + continue + + params = response["params"] + symbol = params[2] + order_book_data = params[1] + timestamp: int = int(time.time()) + + if params[0] is True: + orderbook_msg: OrderBookMessage = DigifinexOrderBook.snapshot_message_from_exchange( + order_book_data, + timestamp, + metadata={"trading_pair": digifinex_utils.convert_from_ws_trading_pair(symbol)} + ) + else: + orderbook_msg: OrderBookMessage = DigifinexOrderBook.diff_message_from_exchange( + order_book_data, + timestamp, + metadata={"trading_pair": digifinex_utils.convert_from_ws_trading_pair(symbol)} + ) + output.put_nowait(orderbook_msg) + + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unexpected error with WebSocket connection.", + exc_info=True, + app_warning_msg="Unexpected error with WebSocket connection. Retrying in 30 seconds. " + "Check network connection." + ) + await asyncio.sleep(30.0) + finally: + await ws.disconnect() + + async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for orderbook snapshots by fetching orderbook + """ + while True: + try: + for trading_pair in self._trading_pairs: + try: + snapshot: Dict[str, any] = await self.get_order_book_data(trading_pair) + snapshot_timestamp: int = snapshot["date"] + snapshot_msg: OrderBookMessage = DigifinexOrderBook.snapshot_message_from_exchange( + snapshot, + snapshot_timestamp, + metadata={"trading_pair": trading_pair} + ) + output.put_nowait(snapshot_msg) + self.logger().debug(f"Saved order book snapshot for {trading_pair}") + # Be careful not to go above API rate limits. + await asyncio.sleep(5.0) + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().network( + f"Unexpected error with WebSocket connection: {e}", + exc_info=True, + app_warning_msg="Unexpected error with WebSocket connection. Retrying in 5 seconds. " + "Check network connection.\n" + + traceback.format_exc() + ) + await asyncio.sleep(5.0) + this_hour: pd.Timestamp = pd.Timestamp.utcnow().replace(minute=0, second=0, microsecond=0) + next_hour: pd.Timestamp = this_hour + pd.Timedelta(hours=1) + delta: float = next_hour.timestamp() - time.time() + await asyncio.sleep(delta) + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error.", exc_info=True) + await asyncio.sleep(5.0) diff --git a/hummingbot/connector/exchange/digifinex/digifinex_api_user_stream_data_source.py b/hummingbot/connector/exchange/digifinex/digifinex_api_user_stream_data_source.py new file mode 100644 index 0000000000..8059190bc5 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_api_user_stream_data_source.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python + +import time +import asyncio +import logging +from typing import Optional, List, AsyncIterable, Any +from hummingbot.connector.exchange.digifinex.digifinex_global import DigifinexGlobal +from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource +from hummingbot.logger import HummingbotLogger +# from .digifinex_auth import DigifinexAuth +from hummingbot.connector.exchange.digifinex.digifinex_websocket import DigifinexWebsocket +from hummingbot.connector.exchange.digifinex import digifinex_utils + + +class DigifinexAPIUserStreamDataSource(UserStreamTrackerDataSource): + MAX_RETRIES = 20 + MESSAGE_TIMEOUT = 30.0 + + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, _global: DigifinexGlobal, trading_pairs: Optional[List[str]] = []): + self._global: DigifinexGlobal = _global + self._trading_pairs = trading_pairs + self._current_listen_key = None + self._listen_for_user_stream_task = None + self._last_recv_time: float = 0 + super().__init__() + + @property + def last_recv_time(self) -> float: + return self._last_recv_time + + async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: + """ + Subscribe to active orders via web socket + """ + + try: + ws = DigifinexWebsocket(self._global.auth) + await ws.connect() + await ws.subscribe("order", list(map( + lambda pair: f"{digifinex_utils.convert_to_ws_trading_pair(pair)}", + self._trading_pairs + ))) + + currencies = set() + for trade_pair in self._trading_pairs: + trade_pair_currencies = trade_pair.split('-') + currencies.update(trade_pair_currencies) + await ws.subscribe("balance", currencies) + + balance_snapshot = await self._global.rest_api.get_balance() + # { + # "code": 0, + # "list": [ + # { + # "currency": "BTC", + # "free": 4723846.89208129, + # "total": 0 + # } + # ] + # } + yield {'method': 'balance.update', 'params': balance_snapshot['list']} + self._last_recv_time = time.time() + + # await ws.subscribe(["user.order", "user.trade", "user.balance"]) + async for msg in ws.on_message(): + # { + # "method": "balance.update", + # "params": [{ + # "currency": "USDT", + # "free": "99944652.8478545303601106", + # "total": "99944652.8478545303601106", + # "used": "0.0000000000" + # }], + # "id": null + # } + yield msg + self._last_recv_time = time.time() + if (msg.get("result") is None): + continue + except Exception as e: + self.logger().exception(e) + raise e + finally: + await ws.disconnect() + await asyncio.sleep(5) + + async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue) -> AsyncIterable[Any]: + """ + *required + Subscribe to user stream via web socket, and keep the connection open for incoming messages + :param ev_loop: ev_loop to execute this function in + :param output: an async queue where the incoming messages are stored + """ + + while True: + try: + async for msg in self._listen_to_orders_trades_balances(): + output.put_nowait(msg) + except asyncio.CancelledError: + raise + except Exception: + self.logger().error( + "Unexpected error with Digifinex WebSocket connection. " "Retrying after 30 seconds...", exc_info=True + ) + await asyncio.sleep(30.0) diff --git a/hummingbot/connector/exchange/digifinex/digifinex_auth.py b/hummingbot/connector/exchange/digifinex/digifinex_auth.py new file mode 100644 index 0000000000..d90e2da161 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_auth.py @@ -0,0 +1,77 @@ +import hmac +import hashlib +import base64 +import urllib +import aiohttp +from typing import List, Dict, Any +# from hummingbot.connector.exchange.digifinex.digifinex_utils import get_ms_timestamp +from hummingbot.connector.exchange.digifinex import digifinex_constants as Constants +from hummingbot.connector.exchange.digifinex.time_patcher import TimePatcher +# import time + +_time_patcher: TimePatcher = None + + +def time_patcher() -> TimePatcher: + global _time_patcher + if _time_patcher is None: + _time_patcher = TimePatcher('Digifinex', DigifinexAuth.query_time_func) + _time_patcher.start() + return _time_patcher + + +class DigifinexAuth(): + """ + Auth class required by digifinex API + Learn more at https://docs.digifinex.io/en-ww/v3/#signature-authentication-amp-verification + """ + def __init__(self, api_key: str, secret_key: str): + self.api_key = api_key + self.secret_key = secret_key + self.time_patcher = time_patcher() + # self.time_patcher = time + + @classmethod + async def query_time_func() -> float: + async with aiohttp.ClientSession() as session: + async with session.get(Constants.REST_URL + '/time') as resp: + resp_data: Dict[str, float] = await resp.json() + return float(resp_data["server_time"]) + + def get_private_headers( + self, + path_url: str, + request_id: int, + data: Dict[str, Any] = None + ): + + data = data or {} + payload = urllib.parse.urlencode(data) + sig = hmac.new( + self.secret_key.encode('utf-8'), + payload.encode('utf-8'), + hashlib.sha256 + ).hexdigest() + nonce = int(self.time_patcher.time()) + + header = { + 'ACCESS-KEY': self.api_key, + 'ACCESS-TIMESTAMP': str(nonce), + 'ACCESS-SIGN': sig, + } + + return header + + def generate_ws_signature(self) -> List[Any]: + data = [None] * 3 + data[0] = self.api_key + nonce = int(self.time_patcher.time() * 1000) + data[1] = str(nonce) + + data[2] = base64.b64encode(hmac.new( + self.secret_key.encode('latin-1'), + f"{nonce}".encode('latin-1'), + hashlib.sha256 + ).digest()) + + return data diff --git a/hummingbot/connector/exchange/digifinex/digifinex_constants.py b/hummingbot/connector/exchange/digifinex/digifinex_constants.py new file mode 100644 index 0000000000..0caa476ba7 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_constants.py @@ -0,0 +1,39 @@ +# A single source of truth for constant variables related to the exchange +import os + +if os.environ.get('digifinex_test') == '1': + host = 'openapi.digifinex.vip' +else: + host = 'openapi.digifinex.com' + +EXCHANGE_NAME = "digifinex" +REST_URL = f"https://{host}/v3" +WSS_PRIVATE_URL = f"wss://{host}/ws/v1/" +WSS_PUBLIC_URL = f"wss://{host}/ws/v1/" + +API_REASONS = { + 0: "Success", + 10001: "Malformed request, (E.g. not using application/json for REST)", + 10002: "Not authenticated, or key/signature incorrect", + 10003: "IP address not whitelisted", + 10004: "Missing required fields", + 10005: "Disallowed based on user tier", + 10006: "Requests have exceeded rate limits", + 10007: "Nonce value differs by more than 30 seconds from server", + 10008: "Invalid method specified", + 10009: "Invalid date range", + 20001: "Duplicated record", + 20002: "Insufficient balance", + 30003: "Invalid instrument_name specified", + 30004: "Invalid side specified", + 30005: "Invalid type specified", + 30006: "Price is lower than the minimum", + 30007: "Price is higher than the maximum", + 30008: "Quantity is lower than the minimum", + 30009: "Quantity is higher than the maximum", + 30010: "Required argument is blank or missing", + 30013: "Too many decimal places for Price", + 30014: "Too many decimal places for Quantity", + 30016: "The notional amount is less than the minimum", + 30017: "The notional amount exceeds the maximum", +} diff --git a/hummingbot/connector/exchange/digifinex/digifinex_exchange.py b/hummingbot/connector/exchange/digifinex/digifinex_exchange.py new file mode 100644 index 0000000000..db01913616 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_exchange.py @@ -0,0 +1,829 @@ +import logging +from typing import ( + Dict, + List, + Optional, + Any, + AsyncIterable, +) +from decimal import Decimal +import asyncio +# import json +# import aiohttp +import math +import time + +from hummingbot.core.network_iterator import NetworkStatus +from hummingbot.logger import HummingbotLogger +from hummingbot.core.clock import Clock +from hummingbot.core.utils import estimate_fee +from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather +from hummingbot.connector.trading_rule import TradingRule +from hummingbot.core.data_type.cancellation_result import CancellationResult +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.limit_order import LimitOrder +from hummingbot.core.event.events import ( + MarketEvent, + BuyOrderCompletedEvent, + SellOrderCompletedEvent, + OrderFilledEvent, + OrderCancelledEvent, + BuyOrderCreatedEvent, + SellOrderCreatedEvent, + MarketOrderFailureEvent, + OrderType, + TradeType, + TradeFee +) +from hummingbot.connector.exchange_base import ExchangeBase +from hummingbot.connector.exchange.digifinex.digifinex_global import DigifinexGlobal +from hummingbot.connector.exchange.digifinex.digifinex_order_book_tracker import DigifinexOrderBookTracker +from hummingbot.connector.exchange.digifinex.digifinex_user_stream_tracker import DigifinexUserStreamTracker +# from hummingbot.connector.exchange.digifinex.digifinex_auth import DigifinexAuth +from hummingbot.connector.exchange.digifinex.digifinex_in_flight_order import DigifinexInFlightOrder +from hummingbot.connector.exchange.digifinex import digifinex_utils +# from hummingbot.connector.exchange.digifinex import digifinex_constants as Constants +from hummingbot.core.data_type.common import OpenOrder +ctce_logger = None +s_decimal_NaN = Decimal("nan") + + +class DigifinexExchange(ExchangeBase): + """ + DigifinexExchange connects with digifinex.com exchange and provides order book pricing, user account tracking and + trading functionality. + """ + API_CALL_TIMEOUT = 10.0 + SHORT_POLL_INTERVAL = 5.0 + UPDATE_ORDER_STATUS_MIN_INTERVAL = 10.0 + LONG_POLL_INTERVAL = 120.0 + + @classmethod + def logger(cls) -> HummingbotLogger: + global ctce_logger + if ctce_logger is None: + ctce_logger = logging.getLogger(__name__) + return ctce_logger + + def __init__(self, + digifinex_api_key: str, + digifinex_secret_key: str, + trading_pairs: Optional[List[str]] = None, + trading_required: bool = True + ): + """ + :param key: The API key to connect to private digifinex.com APIs. + :param secret: The API secret. + :param trading_pairs: The market trading pairs which to track order book data. + :param trading_required: Whether actual trading is needed. + """ + super().__init__() + self._trading_required = trading_required + self._trading_pairs = trading_pairs + self._global = DigifinexGlobal(digifinex_api_key, digifinex_secret_key) + # self._rest_api = DigifinexRestApi(self._digifinex_auth, self._http_client) + self._order_book_tracker = DigifinexOrderBookTracker(trading_pairs=trading_pairs) + self._user_stream_tracker = DigifinexUserStreamTracker(self._global, trading_pairs) + self._ev_loop = asyncio.get_event_loop() + self._poll_notifier = asyncio.Event() + self._last_timestamp = 0 + self._in_flight_orders: Dict[str, DigifinexInFlightOrder] = {} # Dict[client_order_id:str, DigifinexInFlightOrder] + self._order_not_found_records = {} # Dict[client_order_id:str, count:int] + self._trading_rules = {} # Dict[trading_pair:str, TradingRule] + self._status_polling_task = None + self._user_stream_event_listener_task = None + self._trading_rules_polling_task = None + self._last_poll_timestamp = 0 + + @property + def name(self) -> str: + return "digifinex" + + @property + def order_books(self) -> Dict[str, OrderBook]: + return self._order_book_tracker.order_books + + @property + def trading_rules(self) -> Dict[str, TradingRule]: + return self._trading_rules + + @property + def in_flight_orders(self) -> Dict[str, DigifinexInFlightOrder]: + return self._in_flight_orders + + @property + def status_dict(self) -> Dict[str, bool]: + """ + A dictionary of statuses of various connector's components. + """ + return { + "order_books_initialized": self._order_book_tracker.ready, + "account_balance": len(self._account_balances) > 0 if self._trading_required else True, + "trading_rule_initialized": len(self._trading_rules) > 0, + "user_stream_initialized": + self._user_stream_tracker.data_source.last_recv_time > 0 if self._trading_required else True, + } + + @property + def ready(self) -> bool: + """ + :return True when all statuses pass, this might take 5-10 seconds for all the connector's components and + services to be ready. + """ + return all(self.status_dict.values()) + + @property + def limit_orders(self) -> List[LimitOrder]: + return [ + in_flight_order.to_limit_order() + for in_flight_order in self._in_flight_orders.values() + ] + + @property + def tracking_states(self) -> Dict[str, any]: + """ + :return active in-flight orders in json format, is used to save in sqlite db. + """ + return { + key: value.to_json() + for key, value in self._in_flight_orders.items() + if not value.is_done + } + + def restore_tracking_states(self, saved_states: Dict[str, any]): + """ + Restore in-flight orders from saved tracking states, this is st the connector can pick up on where it left off + when it disconnects. + :param saved_states: The saved tracking_states. + """ + self._in_flight_orders.update({ + key: DigifinexInFlightOrder.from_json(value) + for key, value in saved_states.items() + }) + + def supported_order_types(self) -> List[OrderType]: + """ + :return a list of OrderType supported by this connector. + Note that Market order type is no longer required and will not be used. + """ + return [OrderType.LIMIT, OrderType.LIMIT_MAKER] + + def start(self, clock: Clock, timestamp: float): + """ + This function is called automatically by the clock. + """ + super().start(clock, timestamp) + + def stop(self, clock: Clock): + """ + This function is called automatically by the clock. + """ + super().stop(clock) + + async def start_network(self): + """ + This function is required by NetworkIterator base class and is called automatically. + It starts tracking order book, polling trading rules, + updating statuses and tracking user data. + """ + self._order_book_tracker.start() + self._trading_rules_polling_task = safe_ensure_future(self._trading_rules_polling_loop()) + if self._trading_required: + self._status_polling_task = safe_ensure_future(self._status_polling_loop()) + self._user_stream_tracker_task = safe_ensure_future(self._user_stream_tracker.start()) + self._user_stream_event_listener_task = safe_ensure_future(self._user_stream_event_listener()) + + async def stop_network(self): + """ + This function is required by NetworkIterator base class and is called automatically. + """ + self._order_book_tracker.stop() + if self._status_polling_task is not None: + self._status_polling_task.cancel() + self._status_polling_task = None + if self._trading_rules_polling_task is not None: + self._trading_rules_polling_task.cancel() + self._trading_rules_polling_task = None + if self._status_polling_task is not None: + self._status_polling_task.cancel() + self._status_polling_task = None + if self._user_stream_tracker_task is not None: + self._user_stream_tracker_task.cancel() + self._user_stream_tracker_task = None + if self._user_stream_event_listener_task is not None: + self._user_stream_event_listener_task.cancel() + self._user_stream_event_listener_task = None + + async def check_network(self) -> NetworkStatus: + """ + This function is required by NetworkIterator base class and is called periodically to check + the network connection. Simply ping the network (or call any light weight public API). + """ + try: + await self._global.rest_api.request("get", "ping") + except asyncio.CancelledError: + raise + except Exception as e: + _ = e + self.logger().exception('check_network', stack_info=True) + return NetworkStatus.NOT_CONNECTED + return NetworkStatus.CONNECTED + + async def _trading_rules_polling_loop(self): + """ + Periodically update trading rule. + """ + while True: + try: + await self._update_trading_rules() + await asyncio.sleep(60) + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().network(f"Unexpected error while fetching trading rules. Error: {str(e)}", + exc_info=True, + app_warning_msg="Could not fetch new trading rules from digifinex.com. " + "Check network connection.") + await asyncio.sleep(0.5) + + async def _update_trading_rules(self): + instruments_info = await self._global.rest_api.request("get", path_url="markets") + self._trading_rules.clear() + self._trading_rules = self._format_trading_rules(instruments_info) + + def _format_trading_rules(self, instruments_info: Dict[str, Any]) -> Dict[str, TradingRule]: + """ + Converts json API response into a dictionary of trading rules. + :param instruments_info: The json API response + :return A dictionary of trading rules. + Response Example: + { + "data": [{ + "volume_precision": 4, + "price_precision": 2, + "market": "btc_usdt", + "min_amount": 2, + "min_volume": 0.0001 + }], + "date": 1589873858, + "code": 0 + } + """ + result = {} + for rule in instruments_info["data"]: + try: + trading_pair = digifinex_utils.convert_from_exchange_trading_pair(rule["market"]) + price_decimals = Decimal(str(rule["price_precision"])) + quantity_decimals = Decimal(str(rule["volume_precision"])) + # E.g. a price decimal of 2 means 0.01 incremental. + price_step = Decimal("1") / Decimal(str(math.pow(10, price_decimals))) + quantity_step = Decimal("1") / Decimal(str(math.pow(10, quantity_decimals))) + result[trading_pair] = TradingRule(trading_pair, + min_price_increment=price_step, + min_base_amount_increment=quantity_step) + except Exception: + self.logger().error(f"Error parsing the trading pair rule {rule}. Skipping.", exc_info=True) + return result + + def get_order_price_quantum(self, trading_pair: str, price: Decimal): + """ + Returns a price step, a minimum price increment for a given trading pair. + """ + trading_rule = self._trading_rules[trading_pair] + return trading_rule.min_price_increment + + def get_order_size_quantum(self, trading_pair: str, order_size: Decimal): + """ + Returns an order amount step, a minimum amount increment for a given trading pair. + """ + trading_rule = self._trading_rules[trading_pair] + return Decimal(trading_rule.min_base_amount_increment) + + def get_order_book(self, trading_pair: str) -> OrderBook: + if trading_pair not in self._order_book_tracker.order_books: + raise ValueError(f"No order book exists for '{trading_pair}'.") + return self._order_book_tracker.order_books[trading_pair] + + def buy(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, + price: Decimal = s_decimal_NaN, **kwargs) -> str: + """ + Buys an amount of base asset (of the given trading pair). This function returns immediately. + To see an actual order, you'll have to wait for BuyOrderCreatedEvent. + :param trading_pair: The market (e.g. BTC-USDT) to buy from + :param amount: The amount in base token value + :param order_type: The order type + :param price: The price (note: this is no longer optional) + :returns A new internal order id + """ + order_id: str = digifinex_utils.get_new_client_order_id(True, trading_pair) + safe_ensure_future(self._create_order(TradeType.BUY, order_id, trading_pair, amount, order_type, price)) + return order_id + + def sell(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, + price: Decimal = s_decimal_NaN, **kwargs) -> str: + """ + Sells an amount of base asset (of the given trading pair). This function returns immediately. + To see an actual order, you'll have to wait for SellOrderCreatedEvent. + :param trading_pair: The market (e.g. BTC-USDT) to sell from + :param amount: The amount in base token value + :param order_type: The order type + :param price: The price (note: this is no longer optional) + :returns A new internal order id + """ + order_id: str = digifinex_utils.get_new_client_order_id(False, trading_pair) + safe_ensure_future(self._create_order(TradeType.SELL, order_id, trading_pair, amount, order_type, price)) + return order_id + + def cancel(self, trading_pair: str, order_id: str): + """ + Cancel an order. This function returns immediately. + To get the cancellation result, you'll have to wait for OrderCancelledEvent. + :param trading_pair: The market (e.g. BTC-USDT) of the order. + :param order_id: The internal order id (also called client_order_id) + """ + tracked_order = self._in_flight_orders.get(order_id) + if tracked_order is None: + raise ValueError(f"Failed to cancel order - {order_id}. Order not found.") + if tracked_order.exchange_order_id is None: + self.ev_loop.run_until_complete(tracked_order.get_exchange_order_id()) + safe_ensure_future(self._execute_cancel(tracked_order)) + return order_id + + async def _create_order(self, + trade_type: TradeType, + order_id: str, + trading_pair: str, + amount: Decimal, + order_type: OrderType, + price: Decimal): + """ + Calls create-order API end point to place an order, starts tracking the order and triggers order created event. + :param trade_type: BUY or SELL + :param order_id: Internal order id (also called client_order_id) + :param trading_pair: The market to place order + :param amount: The order amount (in base token value) + :param order_type: The order type + :param price: The order price + """ + if not order_type.is_limit_type(): + raise Exception(f"Unsupported order type: {order_type}") + trading_rule = self._trading_rules[trading_pair] + + amount = self.quantize_order_amount(trading_pair, amount) + price = self.quantize_order_price(trading_pair, price) + if amount < trading_rule.min_order_size: + raise ValueError(f"Buy order amount {amount} is lower than the minimum order size " + f"{trading_rule.min_order_size}.") + symbol = digifinex_utils.convert_to_exchange_trading_pair(trading_pair) + api_params = {"symbol": symbol, + "type": trade_type.name.lower(), + "price": f"{price:f}", + "amount": f"{amount:f}", + # "client_oid": order_id + } + if order_type is OrderType.LIMIT_MAKER: + api_params["post_only"] = 1 + self.start_tracking_order(order_id, + None, + trading_pair, + trade_type, + price, + amount, + order_type + ) + try: + order_result = await self._global.rest_api.request("post", "spot/order/new", api_params, True) + exchange_order_id = str(order_result["order_id"]) + tracked_order = self._in_flight_orders.get(order_id) + if tracked_order is not None: + self.logger().info(f"Created {order_type.name} {trade_type.name} order {order_id} for " + f"{amount} {trading_pair}.") + tracked_order.update_exchange_order_id(exchange_order_id) + + event_tag = MarketEvent.BuyOrderCreated if trade_type is TradeType.BUY else MarketEvent.SellOrderCreated + event_class = BuyOrderCreatedEvent if trade_type is TradeType.BUY else SellOrderCreatedEvent + self.trigger_event(event_tag, + event_class( + self.current_timestamp, + order_type, + trading_pair, + amount, + price, + order_id + )) + except asyncio.CancelledError: + raise + except Exception as e: + self.stop_tracking_order(order_id) + self.logger().network( + f"Error submitting {trade_type.name} {order_type.name} order to Digifinex for " + f"{amount} {trading_pair} " + f"{price}.", + exc_info=True, + app_warning_msg=str(e) + ) + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent(self.current_timestamp, order_id, order_type)) + + def start_tracking_order(self, + order_id: str, + exchange_order_id: str, + trading_pair: str, + trade_type: TradeType, + price: Decimal, + amount: Decimal, + order_type: OrderType): + """ + Starts tracking an order by simply adding it into _in_flight_orders dictionary. + """ + self._in_flight_orders[order_id] = DigifinexInFlightOrder( + client_order_id=order_id, + exchange_order_id=exchange_order_id, + trading_pair=trading_pair, + order_type=order_type, + trade_type=trade_type, + price=price, + amount=amount + ) + + def stop_tracking_order(self, order_id: str): + """ + Stops tracking an order by simply removing it from _in_flight_orders dictionary. + """ + if order_id in self._in_flight_orders: + del self._in_flight_orders[order_id] + + async def _execute_cancel(self, o: DigifinexInFlightOrder) -> str: + """ + Executes order cancellation process by first calling cancel-order API. The API result doesn't confirm whether + the cancellation is successful, it simply states it receives the request. + :param trading_pair: The market trading pair + :param order_id: The internal order id + order.last_state to change to CANCELED + """ + try: + await self._global.rest_api.request( + "post", + "spot/order/cancel", + {"order_id": o.exchange_order_id}, + True + ) + if o.client_order_id in self._in_flight_orders: + self.trigger_event(MarketEvent.OrderCancelled, + OrderCancelledEvent(self.current_timestamp, o.client_order_id)) + del self._in_flight_orders[o.client_order_id] + return o.exchange_order_id + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().network( + f"Failed to cancel order {o.exchange_order_id}: {str(e)}", + exc_info=True, + app_warning_msg=f"Failed to cancel the order {o.exchange_order_id} on Digifinex. " + f"Check API key and network connection." + ) + + async def _status_polling_loop(self): + """ + Periodically update user balances and order status via REST API. This serves as a fallback measure for web + socket API updates. + """ + while True: + try: + self._poll_notifier = asyncio.Event() + await self._poll_notifier.wait() + await safe_gather( + self._update_balances(), + self._update_order_status(), + ) + self._last_poll_timestamp = self.current_timestamp + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().error(str(e), exc_info=True) + self.logger().network("Unexpected error while fetching account updates.", + exc_info=True, + app_warning_msg="Could not fetch account updates from Digifinex. " + "Check API key and network connection.") + await asyncio.sleep(0.5) + + async def _update_balances(self): + local_asset_names = set(self._account_balances.keys()) + remote_asset_names = set() + account_info = await self._global.rest_api.get_balance() + for account in account_info["list"]: + asset_name = account["currency"] + self._account_available_balances[asset_name] = Decimal(str(account["free"])) + self._account_balances[asset_name] = Decimal(str(account["total"])) + remote_asset_names.add(asset_name) + + try: + asset_names_to_remove = local_asset_names.difference(remote_asset_names) + for asset_name in asset_names_to_remove: + del self._account_available_balances[asset_name] + del self._account_balances[asset_name] + except Exception as e: + self.logger().error(e) + + async def _update_order_status(self): + """ + Calls REST API to get status update for each in-flight order. + """ + last_tick = int(self._last_poll_timestamp / self.UPDATE_ORDER_STATUS_MIN_INTERVAL) + current_tick = int(self.current_timestamp / self.UPDATE_ORDER_STATUS_MIN_INTERVAL) + + if current_tick > last_tick and len(self._in_flight_orders) > 0: + tracked_orders = list(self._in_flight_orders.values()) + tasks = [] + for tracked_order in tracked_orders: + order_id = await tracked_order.get_exchange_order_id() + tasks.append(self._global.rest_api.request("get", + "spot/order/detail", + {"order_id": order_id}, + True)) + self.logger().debug(f"Polling for order status updates of {len(tasks)} orders.") + update_results = await safe_gather(*tasks, return_exceptions=True) + for update_result in update_results: + if isinstance(update_result, Exception): + raise update_result + if "data" not in update_result: + self.logger().info(f"_update_order_status result not in resp: {update_result}") + continue + order_data = update_result["data"] + self._process_rest_trade_details(order_data) + self._process_order_status(order_data.get('order_id'), order_data.get('status')) + + def _process_order_status(self, exchange_order_id: str, status: int): + """ + Updates in-flight order and triggers cancellation or failure event if needed. + """ + tracked_order = self.find_exchange_order(exchange_order_id) + if tracked_order is None: + return + client_order_id = tracked_order.client_order_id + # Update order execution status + tracked_order.last_state = str(status) + if tracked_order.is_cancelled: + self.logger().info(f"Successfully cancelled order {client_order_id}.") + self.trigger_event(MarketEvent.OrderCancelled, + OrderCancelledEvent( + self.current_timestamp, + client_order_id)) + tracked_order.cancelled_event.set() + self.stop_tracking_order(client_order_id) + # elif tracked_order.is_failure: + # self.logger().info(f"The market order {client_order_id} has failed according to order status API. " + # f"Reason: {digifinex_utils.get_api_reason(order_msg['reason'])}") + # self.trigger_event(MarketEvent.OrderFailure, + # MarketOrderFailureEvent( + # self.current_timestamp, + # client_order_id, + # tracked_order.order_type + # )) + # self.stop_tracking_order(client_order_id) + + def _process_rest_trade_details(self, order_detail_msg: Any): + for trade_msg in order_detail_msg['detail']: + """ + Updates in-flight order and trigger order filled event for trade message received. Triggers order completed + event if the total executed amount equals to the specified order amount. + """ + # for order in self._in_flight_orders.values(): + # await order.get_exchange_order_id() + tracked_order = self.find_exchange_order(trade_msg['order_id']) + if tracked_order is None: + return + + updated = tracked_order.update_with_rest_order_detail(trade_msg) + if not updated: + return + + self.trigger_event( + MarketEvent.OrderFilled, + OrderFilledEvent( + self.current_timestamp, + tracked_order.client_order_id, + tracked_order.trading_pair, + tracked_order.trade_type, + tracked_order.order_type, + Decimal(str(trade_msg["executed_price"])), + Decimal(str(trade_msg["executed_amount"])), + estimate_fee.estimate_fee(self.name, tracked_order.order_type in [OrderType.LIMIT, OrderType.LIMIT_MAKER]), + # TradeFee(0.0, [(trade_msg["fee_currency"], Decimal(str(trade_msg["fee"])))]), + exchange_trade_id=trade_msg["tid"] + ) + ) + if math.isclose(tracked_order.executed_amount_base, tracked_order.amount) or \ + tracked_order.executed_amount_base >= tracked_order.amount: + tracked_order.last_state = "FILLED" + self.logger().info(f"The {tracked_order.trade_type.name} order " + f"{tracked_order.client_order_id} has completed " + f"according to order status API.") + event_tag = MarketEvent.BuyOrderCompleted if tracked_order.trade_type is TradeType.BUY \ + else MarketEvent.SellOrderCompleted + event_class = BuyOrderCompletedEvent if tracked_order.trade_type is TradeType.BUY \ + else SellOrderCompletedEvent + self.trigger_event(event_tag, + event_class(self.current_timestamp, + tracked_order.client_order_id, + tracked_order.base_asset, + tracked_order.quote_asset, + tracked_order.fee_asset, + tracked_order.executed_amount_base, + tracked_order.executed_amount_quote, + tracked_order.fee_paid, + tracked_order.order_type)) + self.stop_tracking_order(tracked_order.client_order_id) + + def find_exchange_order(self, exchange_order_id: str): + for o in self._in_flight_orders.values(): + if o.exchange_order_id == exchange_order_id: + return o + + def _process_order_message_traded(self, order_msg): + tracked_order: DigifinexInFlightOrder = self.find_exchange_order(order_msg['id']) + if tracked_order is None: + return + + (delta_trade_amount, delta_trade_price) = tracked_order.update_with_order_update(order_msg) + if not delta_trade_amount: + return + + self.trigger_event( + MarketEvent.OrderFilled, + OrderFilledEvent( + self.current_timestamp, + tracked_order.client_order_id, + tracked_order.trading_pair, + tracked_order.trade_type, + tracked_order.order_type, + delta_trade_price, + delta_trade_amount, + estimate_fee.estimate_fee(self.name, tracked_order.order_type in [OrderType.LIMIT, OrderType.LIMIT_MAKER]), + # TradeFee(0.0, [(trade_msg["fee_currency"], Decimal(str(trade_msg["fee"])))]), + exchange_trade_id='N/A' + ) + ) + if math.isclose(tracked_order.executed_amount_base, tracked_order.amount) or \ + tracked_order.executed_amount_base >= tracked_order.amount: + tracked_order.last_state = "2" + self.logger().info(f"The {tracked_order.trade_type.name} order " + f"{tracked_order.client_order_id} has completed " + f"according to order status API.") + event_tag = MarketEvent.BuyOrderCompleted if tracked_order.trade_type is TradeType.BUY \ + else MarketEvent.SellOrderCompleted + event_class = BuyOrderCompletedEvent if tracked_order.trade_type is TradeType.BUY \ + else SellOrderCompletedEvent + self.trigger_event(event_tag, + event_class(self.current_timestamp, + tracked_order.client_order_id, + tracked_order.base_asset, + tracked_order.quote_asset, + tracked_order.fee_asset, + tracked_order.executed_amount_base, + tracked_order.executed_amount_quote, + tracked_order.fee_paid, + tracked_order.order_type)) + self.stop_tracking_order(tracked_order.client_order_id) + + async def cancel_all(self, timeout_seconds: float): + """ + Cancels all in-flight orders and waits for cancellation results. + Used by bot's top level stop and exit commands (cancelling outstanding orders on exit) + :param timeout_seconds: The timeout at which the operation will be canceled. + :returns List of CancellationResult which indicates whether each order is successfully cancelled. + """ + if self._trading_pairs is None: + raise Exception("cancel_all can only be used when trading_pairs are specified.") + cancellation_results = [] + try: + # for trading_pair in self._trading_pairs: + # await self._global.rest_api.request( + # "post", + # "private/cancel-all-orders", + # {"instrument_name": digifinex_utils.convert_to_exchange_trading_pair(trading_pair)}, + # True + # ) + + open_orders = list(self._in_flight_orders.values()) + for o in open_orders: + await self._execute_cancel(o) + + for cl_order_id, tracked_order in self._in_flight_orders.items(): + open_order = [o for o in open_orders if o.exchange_order_id == tracked_order.exchange_order_id] + if not open_order: + cancellation_results.append(CancellationResult(cl_order_id, True)) + # self.trigger_event(MarketEvent.OrderCancelled, + # OrderCancelledEvent(self.current_timestamp, cl_order_id)) + else: + cancellation_results.append(CancellationResult(cl_order_id, False)) + except Exception: + self.logger().network( + "Failed to cancel all orders.", + exc_info=True, + app_warning_msg="Failed to cancel all orders on Digifinex. Check API key and network connection." + ) + return cancellation_results + + def tick(self, timestamp: float): + """ + Is called automatically by the clock for each clock's tick (1 second by default). + It checks if status polling task is due for execution. + """ + now = time.time() + poll_interval = (self.SHORT_POLL_INTERVAL + if now - self._user_stream_tracker.last_recv_time > 60.0 + else self.LONG_POLL_INTERVAL) + last_tick = int(self._last_timestamp / poll_interval) + current_tick = int(timestamp / poll_interval) + if current_tick > last_tick: + if not self._poll_notifier.is_set(): + self._poll_notifier.set() + self._last_timestamp = timestamp + + def get_fee(self, + base_currency: str, + quote_currency: str, + order_type: OrderType, + order_side: TradeType, + amount: Decimal, + price: Decimal = s_decimal_NaN) -> TradeFee: + """ + To get trading fee, this function is simplified by using fee override configuration. Most parameters to this + function are ignore except order_type. Use OrderType.LIMIT_MAKER to specify you want trading fee for + maker order. + """ + is_maker = order_type is OrderType.LIMIT_MAKER + return TradeFee(percent=self.estimate_fee_pct(is_maker)) + + async def _iter_user_event_queue(self) -> AsyncIterable[Dict[str, any]]: + while True: + try: + yield await self._user_stream_tracker.user_stream.get() + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unknown error. Retrying after 1 seconds.", + exc_info=True, + app_warning_msg="Could not fetch user events from Digifinex. Check API key and network connection." + ) + await asyncio.sleep(1.0) + + async def _user_stream_event_listener(self): + """ + Listens to message in _user_stream_tracker.user_stream queue. The messages are put in by + DigifinexAPIUserStreamDataSource. + """ + async for event_message in self._iter_user_event_queue(): + try: + if "method" not in event_message: + continue + channel = event_message["method"] + # if "user.trade" in channel: + # for trade_msg in event_message["result"]["data"]: + # await self._process_trade_message(trade_msg) + if "order.update" in channel: + for order_msg in event_message["params"]: + self._process_order_status(order_msg['id'], order_msg['status']) + self._process_order_message_traded(order_msg) + elif channel == "balance.update": + balances = event_message["params"] + for balance_entry in balances: + asset_name = balance_entry["currency"] + self._account_balances[asset_name] = Decimal(str(balance_entry["total"])) + self._account_available_balances[asset_name] = Decimal(str(balance_entry["free"])) + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error in user stream listener loop.", exc_info=True) + await asyncio.sleep(5.0) + + async def get_open_orders(self) -> List[OpenOrder]: + result = await self._global.rest_api.request( + "get", + "spot/order/current", + {}, + True + ) + ret_val = [] + for order in result["data"]: + # if digifinex_utils.HBOT_BROKER_ID not in order["client_oid"]: + # continue + if order["type"] not in ["buy", "sell"]: + raise Exception(f"Unsupported order type {order['type']}") + ret_val.append( + OpenOrder( + client_order_id=None, + trading_pair=digifinex_utils.convert_from_exchange_trading_pair(order["symbol"]), + price=Decimal(str(order["price"])), + amount=Decimal(str(order["amount"])), + executed_amount=Decimal(str(order["executed_amount"])), + status=order["status"], + order_type=OrderType.LIMIT, + is_buy=True if order["type"] == "buy" else False, + time=int(order["created_date"]), + exchange_order_id=order["order_id"] + ) + ) + return ret_val diff --git a/hummingbot/connector/exchange/digifinex/digifinex_global.py b/hummingbot/connector/exchange/digifinex/digifinex_global.py new file mode 100644 index 0000000000..13b8350d96 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_global.py @@ -0,0 +1,19 @@ +import aiohttp +from hummingbot.connector.exchange.digifinex.digifinex_auth import DigifinexAuth +from hummingbot.connector.exchange.digifinex.digifinex_rest_api import DigifinexRestApi + + +class DigifinexGlobal: + + def __init__(self, key: str, secret: str): + self.auth = DigifinexAuth(key, secret) + self.rest_api = DigifinexRestApi(self.auth, self.http_client) + self._shared_client: aiohttp.ClientSession = None + + async def http_client(self) -> aiohttp.ClientSession: + """ + :returns Shared client session instance + """ + if self._shared_client is None: + self._shared_client = aiohttp.ClientSession() + return self._shared_client diff --git a/hummingbot/connector/exchange/digifinex/digifinex_in_flight_order.py b/hummingbot/connector/exchange/digifinex/digifinex_in_flight_order.py new file mode 100644 index 0000000000..4c7382acee --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_in_flight_order.py @@ -0,0 +1,123 @@ +from decimal import Decimal +from typing import ( + Any, + Dict, + Optional, + Tuple, +) +import asyncio +from hummingbot.core.event.events import ( + OrderType, + TradeType +) +from hummingbot.connector.in_flight_order_base import InFlightOrderBase + + +class DigifinexInFlightOrder(InFlightOrderBase): + def __init__(self, + client_order_id: str, + exchange_order_id: Optional[str], + trading_pair: str, + order_type: OrderType, + trade_type: TradeType, + price: Decimal, + amount: Decimal, + initial_state: str = "OPEN"): + super().__init__( + client_order_id, + exchange_order_id, + trading_pair, + order_type, + trade_type, + price, + amount, + initial_state, + ) + self.trade_id_set = set() + self.cancelled_event = asyncio.Event() + + @property + def is_done(self) -> bool: + return self.last_state in {"2", "3", "4"} + + @property + def is_failure(self) -> bool: + return False + # return self.last_state in {"REJECTED"} + + @property + def is_cancelled(self) -> bool: + return self.last_state in {"3", "4"} + + # @property + # def order_type_description(self) -> str: + # """ + # :return: Order description string . One of ["limit buy" / "limit sell" / "market buy" / "market sell"] + # """ + # order_type = "market" if self.order_type is OrderType.MARKET else "limit" + # side = "buy" if self.trade_type == TradeType.BUY else "sell" + # return f"{order_type} {side}" + + @classmethod + def from_json(cls, data: Dict[str, Any]) -> InFlightOrderBase: + """ + :param data: json data from API + :return: formatted InFlightOrder + """ + retval = DigifinexInFlightOrder( + data["client_order_id"], + data["exchange_order_id"], + data["trading_pair"], + getattr(OrderType, data["order_type"]), + getattr(TradeType, data["trade_type"]), + Decimal(data["price"]), + Decimal(data["amount"]), + data["last_state"] + ) + retval.executed_amount_base = Decimal(data["executed_amount_base"]) + retval.executed_amount_quote = Decimal(data["executed_amount_quote"]) + retval.fee_asset = data["fee_asset"] + retval.fee_paid = Decimal(data["fee_paid"]) + retval.last_state = data["last_state"] + return retval + + def update_with_rest_order_detail(self, trade_update: Dict[str, Any]) -> bool: + """ + Updates the in flight order with trade update (from private/get-order-detail end point) + return: True if the order gets updated otherwise False + """ + trade_id = trade_update["tid"] + # trade_update["orderId"] is type int + if trade_id in self.trade_id_set: + # trade already recorded + return False + self.trade_id_set.add(trade_id) + self.executed_amount_base += Decimal(str(trade_update["executed_amount"])) + # self.fee_paid += Decimal(str(trade_update["fee"])) + self.executed_amount_quote += (Decimal(str(trade_update["executed_price"])) * + Decimal(str(trade_update["executed_amount"]))) + # if not self.fee_asset: + # self.fee_asset = trade_update["fee_currency"] + return True + + def update_with_order_update(self, order_update) -> Tuple[Decimal, Decimal]: + """ + Updates the in flight order with trade update (from order message) + return: (delta_trade_amount, delta_trade_price) + """ + + # todo: order_msg contains no trade_id. may be re-processed + if order_update['filled'] == '0': + return (0, 0) + + self.trade_id_set.add("N/A") + executed_amount_base = Decimal(order_update['filled']) + if executed_amount_base == self.executed_amount_base: + return (0, 0) + delta_trade_amount = executed_amount_base - self.executed_amount_base + self.executed_amount_base = executed_amount_base + + executed_amount_quote = executed_amount_base * Decimal(order_update['price_avg'] or order_update['price']) + delta_trade_price = (executed_amount_quote - self.executed_amount_quote) / delta_trade_amount + self.executed_amount_quote = executed_amount_base * Decimal(order_update['price_avg'] or order_update['price']) + return (delta_trade_amount, delta_trade_price) diff --git a/hummingbot/connector/exchange/digifinex/digifinex_order_book.py b/hummingbot/connector/exchange/digifinex/digifinex_order_book.py new file mode 100644 index 0000000000..29ecae9727 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_order_book.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python + +import logging +import hummingbot.connector.exchange.digifinex.digifinex_constants as constants + +from sqlalchemy.engine import RowProxy +from typing import ( + Optional, + Dict, + List, Any) +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_message import ( + OrderBookMessage, OrderBookMessageType +) +from hummingbot.connector.exchange.digifinex.digifinex_order_book_message import DigifinexOrderBookMessage + +_logger = None + + +class DigifinexOrderBook(OrderBook): + @classmethod + def logger(cls) -> HummingbotLogger: + global _logger + if _logger is None: + _logger = logging.getLogger(__name__) + return _logger + + @classmethod + def snapshot_message_from_exchange(cls, + msg: Dict[str, any], + timestamp: float, + metadata: Optional[Dict] = None): + """ + Convert json snapshot data into standard OrderBookMessage format + :param msg: json snapshot data from live web socket stream + :param timestamp: timestamp attached to incoming data + :return: DigifinexOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + return DigifinexOrderBookMessage( + message_type=OrderBookMessageType.SNAPSHOT, + content=msg, + timestamp=timestamp + ) + + @classmethod + def snapshot_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of snapshot data into standard OrderBookMessage format + :param record: a row of snapshot data from the database + :return: DigifinexOrderBookMessage + """ + return DigifinexOrderBookMessage( + message_type=OrderBookMessageType.SNAPSHOT, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def diff_message_from_exchange(cls, + msg: Dict[str, any], + timestamp: Optional[float] = None, + metadata: Optional[Dict] = None): + """ + Convert json diff data into standard OrderBookMessage format + :param msg: json diff data from live web socket stream + :param timestamp: timestamp attached to incoming data + :return: DigifinexOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + return DigifinexOrderBookMessage( + message_type=OrderBookMessageType.DIFF, + content=msg, + timestamp=timestamp + ) + + @classmethod + def diff_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of diff data into standard OrderBookMessage format + :param record: a row of diff data from the database + :return: DigifinexOrderBookMessage + """ + return DigifinexOrderBookMessage( + message_type=OrderBookMessageType.DIFF, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def trade_message_from_exchange(cls, + msg: Dict[str, Any], + timestamp: Optional[float] = None, + metadata: Optional[Dict] = None): + """ + Convert a trade data into standard OrderBookMessage format + :param record: a trade data from the database + :return: DigifinexOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + msg.update({ + "exchange_order_id": msg.get("id"), + "trade_type": msg.get("type"), + "price": msg.get("price"), + "amount": msg.get("amount"), + }) + + return DigifinexOrderBookMessage( + message_type=OrderBookMessageType.TRADE, + content=msg, + timestamp=timestamp + ) + + @classmethod + def trade_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of trade data into standard OrderBookMessage format + :param record: a row of trade data from the database + :return: DigifinexOrderBookMessage + """ + return DigifinexOrderBookMessage( + message_type=OrderBookMessageType.TRADE, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def from_snapshot(cls, snapshot: OrderBookMessage): + raise NotImplementedError(constants.EXCHANGE_NAME + " order book needs to retain individual order data.") + + @classmethod + def restore_from_snapshot_and_diffs(self, snapshot: OrderBookMessage, diffs: List[OrderBookMessage]): + raise NotImplementedError(constants.EXCHANGE_NAME + " order book needs to retain individual order data.") diff --git a/hummingbot/connector/exchange/digifinex/digifinex_order_book_message.py b/hummingbot/connector/exchange/digifinex/digifinex_order_book_message.py new file mode 100644 index 0000000000..883ea99da7 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_order_book_message.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python + +from typing import ( + Dict, + List, + Optional, +) + +from hummingbot.core.data_type.order_book_row import OrderBookRow +from hummingbot.core.data_type.order_book_message import ( + OrderBookMessage, + OrderBookMessageType, +) + + +class DigifinexOrderBookMessage(OrderBookMessage): + def __new__( + cls, + message_type: OrderBookMessageType, + content: Dict[str, any], + timestamp: Optional[float] = None, + *args, + **kwargs, + ): + if timestamp is None: + if message_type is OrderBookMessageType.SNAPSHOT: + raise ValueError("timestamp must not be None when initializing snapshot messages.") + timestamp = content["timestamp"] + + return super(DigifinexOrderBookMessage, cls).__new__( + cls, message_type, content, timestamp=timestamp, *args, **kwargs + ) + + @property + def update_id(self) -> int: + if self.type in [OrderBookMessageType.DIFF, OrderBookMessageType.SNAPSHOT]: + return int(self.timestamp * 1e3) + else: + return -1 + + @property + def trade_id(self) -> int: + if self.type is OrderBookMessageType.TRADE: + return int(self.timestamp * 1e3) + return -1 + + @property + def trading_pair(self) -> str: + if "trading_pair" in self.content: + return self.content["trading_pair"] + elif "instrument_name" in self.content: + return self.content["instrument_name"] + + @property + def asks(self) -> List[OrderBookRow]: + asks = map(self.content["asks"], lambda ask: {"price": ask[0], "amount": ask[1]}) + + return [ + OrderBookRow(float(price), float(amount), self.update_id) for price, amount in asks + ] + + @property + def bids(self) -> List[OrderBookRow]: + bids = map(self.content["bids"], lambda bid: {"price": bid[0], "amount": bid[1]}) + + return [ + OrderBookRow(float(price), float(amount), self.update_id) for price, amount in bids + ] + + def __eq__(self, other) -> bool: + return self.type == other.type and self.timestamp == other.timestamp + + def __lt__(self, other) -> bool: + if self.timestamp != other.timestamp: + return self.timestamp < other.timestamp + else: + """ + If timestamp is the same, the ordering is snapshot < diff < trade + """ + return self.type.value < other.type.value diff --git a/hummingbot/connector/exchange/digifinex/digifinex_order_book_tracker.py b/hummingbot/connector/exchange/digifinex/digifinex_order_book_tracker.py new file mode 100644 index 0000000000..a90d5fb035 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_order_book_tracker.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +import asyncio +import bisect +import logging +import hummingbot.connector.exchange.digifinex.digifinex_constants as constants +import time + +from collections import defaultdict, deque +from typing import Optional, Dict, List, Deque +from hummingbot.core.data_type.order_book_message import OrderBookMessageType +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.order_book_tracker import OrderBookTracker +from hummingbot.connector.exchange.digifinex.digifinex_order_book_message import DigifinexOrderBookMessage +from hummingbot.connector.exchange.digifinex.digifinex_active_order_tracker import DigifinexActiveOrderTracker +from hummingbot.connector.exchange.digifinex.digifinex_api_order_book_data_source import DigifinexAPIOrderBookDataSource +from hummingbot.connector.exchange.digifinex.digifinex_order_book import DigifinexOrderBook + + +class DigifinexOrderBookTracker(OrderBookTracker): + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, trading_pairs: Optional[List[str]] = None,): + super().__init__(DigifinexAPIOrderBookDataSource(trading_pairs), trading_pairs) + + self._ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + self._order_book_snapshot_stream: asyncio.Queue = asyncio.Queue() + self._order_book_diff_stream: asyncio.Queue = asyncio.Queue() + self._order_book_trade_stream: asyncio.Queue = asyncio.Queue() + self._process_msg_deque_task: Optional[asyncio.Task] = None + self._past_diffs_windows: Dict[str, Deque] = {} + self._order_books: Dict[str, DigifinexOrderBook] = {} + self._saved_message_queues: Dict[str, Deque[DigifinexOrderBookMessage]] = \ + defaultdict(lambda: deque(maxlen=1000)) + self._active_order_trackers: Dict[str, DigifinexActiveOrderTracker] = defaultdict(DigifinexActiveOrderTracker) + self._order_book_stream_listener_task: Optional[asyncio.Task] = None + self._order_book_trade_listener_task: Optional[asyncio.Task] = None + + @property + def exchange_name(self) -> str: + """ + Name of the current exchange + """ + return constants.EXCHANGE_NAME + + async def _track_single_book(self, trading_pair: str): + """ + Update an order book with changes from the latest batch of received messages + """ + past_diffs_window: Deque[DigifinexOrderBookMessage] = deque() + self._past_diffs_windows[trading_pair] = past_diffs_window + + message_queue: asyncio.Queue = self._tracking_message_queues[trading_pair] + order_book: DigifinexOrderBook = self._order_books[trading_pair] + active_order_tracker: DigifinexActiveOrderTracker = self._active_order_trackers[trading_pair] + + last_message_timestamp: float = time.time() + diff_messages_accepted: int = 0 + + while True: + try: + message: DigifinexOrderBookMessage = None + saved_messages: Deque[DigifinexOrderBookMessage] = self._saved_message_queues[trading_pair] + # Process saved messages first if there are any + if len(saved_messages) > 0: + message = saved_messages.popleft() + else: + message = await message_queue.get() + + if message.type is OrderBookMessageType.DIFF: + bids, asks = active_order_tracker.convert_diff_message_to_order_book_row(message) + order_book.apply_diffs(bids, asks, message.update_id) + past_diffs_window.append(message) + while len(past_diffs_window) > self.PAST_DIFF_WINDOW_SIZE: + past_diffs_window.popleft() + diff_messages_accepted += 1 + + # Output some statistics periodically. + now: float = time.time() + if int(now / 60.0) > int(last_message_timestamp / 60.0): + self.logger().debug("Processed %d order book diffs for %s.", + diff_messages_accepted, trading_pair) + diff_messages_accepted = 0 + last_message_timestamp = now + elif message.type is OrderBookMessageType.SNAPSHOT: + past_diffs: List[DigifinexOrderBookMessage] = list(past_diffs_window) + # only replay diffs later than snapshot, first update active order with snapshot then replay diffs + replay_position = bisect.bisect_right(past_diffs, message) + replay_diffs = past_diffs[replay_position:] + s_bids, s_asks = active_order_tracker.convert_snapshot_message_to_order_book_row(message) + order_book.apply_snapshot(s_bids, s_asks, message.update_id) + for diff_message in replay_diffs: + d_bids, d_asks = active_order_tracker.convert_diff_message_to_order_book_row(diff_message) + order_book.apply_diffs(d_bids, d_asks, diff_message.update_id) + + self.logger().debug("Processed order book snapshot for %s.", trading_pair) + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + f"Unexpected error processing order book messages for {trading_pair}.", + exc_info=True, + app_warning_msg="Unexpected error processing order book messages. Retrying after 5 seconds." + ) + await asyncio.sleep(5.0) diff --git a/hummingbot/connector/exchange/digifinex/digifinex_order_book_tracker_entry.py b/hummingbot/connector/exchange/digifinex/digifinex_order_book_tracker_entry.py new file mode 100644 index 0000000000..afc487dc70 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_order_book_tracker_entry.py @@ -0,0 +1,21 @@ +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_tracker_entry import OrderBookTrackerEntry +from hummingbot.connector.exchange.digifinex.digifinex_active_order_tracker import DigifinexActiveOrderTracker + + +class DigifinexOrderBookTrackerEntry(OrderBookTrackerEntry): + def __init__( + self, trading_pair: str, timestamp: float, order_book: OrderBook, active_order_tracker: DigifinexActiveOrderTracker + ): + self._active_order_tracker = active_order_tracker + super(DigifinexOrderBookTrackerEntry, self).__init__(trading_pair, timestamp, order_book) + + def __repr__(self) -> str: + return ( + f"DigifinexOrderBookTrackerEntry(trading_pair='{self._trading_pair}', timestamp='{self._timestamp}', " + f"order_book='{self._order_book}')" + ) + + @property + def active_order_tracker(self) -> DigifinexActiveOrderTracker: + return self._active_order_tracker diff --git a/hummingbot/connector/exchange/digifinex/digifinex_rest_api.py b/hummingbot/connector/exchange/digifinex/digifinex_rest_api.py new file mode 100644 index 0000000000..1a7f89469a --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_rest_api.py @@ -0,0 +1,111 @@ +from typing import Callable, Dict, Any +import aiohttp +import json +import urllib +from hummingbot.connector.exchange.digifinex.digifinex_auth import DigifinexAuth +from hummingbot.connector.exchange.digifinex import digifinex_constants as Constants +from hummingbot.connector.exchange.digifinex import digifinex_utils + + +class DigifinexRestApi: + + def __init__(self, auth: DigifinexAuth, http_client_getter: Callable[[], aiohttp.ClientSession]): + self._auth = auth + self._http_client = http_client_getter + + async def request(self, + method: str, + path_url: str, + params: Dict[str, Any] = {}, + is_auth_required: bool = False) -> Dict[str, Any]: + """ + Sends an aiohttp request and waits for a response. + :param method: The HTTP method, e.g. get or post + :param path_url: The path url or the API end point + :param is_auth_required: Whether an authentication is required, when True the function will add encrypted + signature to the request. + :returns A response in json format. + """ + url = f"{Constants.REST_URL}/{path_url}" + client = await self._http_client() + if is_auth_required: + request_id = digifinex_utils.RequestId.generate_request_id() + headers = self._auth.get_private_headers(path_url, request_id, params) + else: + headers = {} + + if method == "get": + url = f'{url}?{urllib.parse.urlencode(params)}' + response = await client.get(url, headers=headers) + elif method == "post": + response = await client.post(url, data=params, headers=headers) + else: + raise NotImplementedError + + try: + parsed_response = json.loads(await response.text()) + except Exception as e: + raise IOError(f"Error parsing data from {url}. Error: {str(e)}") + if response.status != 200: + raise IOError(f"Error fetching data from {url}. HTTP status is {response.status}. " + f"Message: {parsed_response}") + code = parsed_response["code"] + if code != 0: + msgs = { + 10001: "Wrong request method, please check it's a GET or POST request", + 10002: "Invalid ApiKey", + 10003: "Sign doesn't match", + 10004: "Illegal request parameters", + 10005: "Request frequency exceeds the limit", + 10006: "Unauthorized to execute this request", + 10007: "IP address Unauthorized", + 10008: "Timestamp for this request is invalid", + 10009: "Unexist endpoint or misses ACCESS-KEY, please check endpoint URL", + 10011: "ApiKey expired. Please go to client side to re-create an ApiKey.", + 20002: "Trade of this trading pair is suspended", + 20007: "Price precision error", + 20008: "Amount precision error", + 20009: "Amount is less than the minimum requirement", + 20010: "Cash Amount is less than the minimum requirement", + 20011: "Insufficient balance", + 20012: "Invalid trade type (valid value: buy/sell)", + 20013: "No order info found", + 20014: "Invalid date (Valid format: 2018-07-25)", + 20015: "Date exceeds the limit", + 20018: "Your have been banned for API trading by the system", + 20019: 'Wrong trading pair symbol, correct format:"base_quote", e.g. "btc_usdt"', + 20020: "You have violated the API trading rules and temporarily banned for trading. At present, we have certain restrictions on the user's transaction rate and withdrawal rate.", + 20021: "Invalid currency", + 20022: "The ending timestamp must be larger than the starting timestamp", + 20023: "Invalid transfer type", + 20024: "Invalid amount", + 20025: "This currency is not transferable at the moment", + 20026: "Transfer amount exceed your balance", + 20027: "Abnormal account status", + 20028: "Blacklist for transfer", + 20029: "Transfer amount exceed your daily limit", + 20030: "You have no position on this trading pair", + 20032: "Withdrawal limited", + 20033: "Wrong Withdrawal ID", + 20034: "Withdrawal service of this crypto has been closed", + 20035: "Withdrawal limit", + 20036: "Withdrawal cancellation failed", + 20037: "The withdrawal address, Tag or chain type is not included in the withdrawal management list", + 20038: "The withdrawal address is not on the white list", + 20039: "Can't be canceled in current status", + 20040: "Withdraw too frequently; limitation: 3 times a minute, 100 times a day", + 20041: "Beyond the daily withdrawal limit", + 20042: "Current trading pair does not support API trading", + 50000: "Exception error", + } + raise IOError(f"{url} API call failed, response: {parsed_response} ({msgs[code]})") + # print(f"REQUEST: {method} {path_url} {params}") + # print(f"RESPONSE: {parsed_response}") + return parsed_response + + async def get_balance(self) -> Dict[str, Any]: + """ + Calls REST API to update total and available balances. + """ + account_info = await self.request("get", "spot/assets", {}, True) + return account_info diff --git a/hummingbot/connector/exchange/bitmax/bitmax_user_stream_tracker.py b/hummingbot/connector/exchange/digifinex/digifinex_user_stream_tracker.py similarity index 78% rename from hummingbot/connector/exchange/bitmax/bitmax_user_stream_tracker.py rename to hummingbot/connector/exchange/digifinex/digifinex_user_stream_tracker.py index 8255abdb94..fe0125a6a3 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_user_stream_tracker.py +++ b/hummingbot/connector/exchange/digifinex/digifinex_user_stream_tracker.py @@ -15,13 +15,13 @@ safe_ensure_future, safe_gather, ) -from hummingbot.connector.exchange.bitmax.bitmax_api_user_stream_data_source import \ - BitmaxAPIUserStreamDataSource -from hummingbot.connector.exchange.bitmax.bitmax_auth import BitmaxAuth -from hummingbot.connector.exchange.bitmax.bitmax_constants import EXCHANGE_NAME +from hummingbot.connector.exchange.digifinex.digifinex_api_user_stream_data_source import \ + DigifinexAPIUserStreamDataSource +from hummingbot.connector.exchange.digifinex.digifinex_global import DigifinexGlobal +from hummingbot.connector.exchange.digifinex.digifinex_constants import EXCHANGE_NAME -class BitmaxUserStreamTracker(UserStreamTracker): +class DigifinexUserStreamTracker(UserStreamTracker): _cbpust_logger: Optional[HummingbotLogger] = None @classmethod @@ -31,10 +31,10 @@ def logger(cls) -> HummingbotLogger: return cls._bust_logger def __init__(self, - bitmax_auth: Optional[BitmaxAuth] = None, + _global: DigifinexGlobal, trading_pairs: Optional[List[str]] = []): super().__init__() - self._bitmax_auth: BitmaxAuth = bitmax_auth + self._global: DigifinexGlobal = _global self._trading_pairs: List[str] = trading_pairs self._ev_loop: asyncio.events.AbstractEventLoop = asyncio.get_event_loop() self._data_source: Optional[UserStreamTrackerDataSource] = None @@ -48,8 +48,8 @@ def data_source(self) -> UserStreamTrackerDataSource: :return: OrderBookTrackerDataSource """ if not self._data_source: - self._data_source = BitmaxAPIUserStreamDataSource( - bitmax_auth=self._bitmax_auth, + self._data_source = DigifinexAPIUserStreamDataSource( + self._global, trading_pairs=self._trading_pairs ) return self._data_source diff --git a/hummingbot/connector/exchange/digifinex/digifinex_utils.py b/hummingbot/connector/exchange/digifinex/digifinex_utils.py new file mode 100644 index 0000000000..a48ac5d3b1 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_utils.py @@ -0,0 +1,98 @@ +import math +from typing import Dict, List + +from hummingbot.core.utils.tracking_nonce import get_tracking_nonce, get_tracking_nonce_low_res +from . import digifinex_constants as Constants + +from hummingbot.client.config.config_var import ConfigVar +from hummingbot.client.config.config_methods import using_exchange + + +CENTRALIZED = True + +EXAMPLE_PAIR = "ETH-USDT" + +DEFAULT_FEES = [0.1, 0.1] + +HBOT_BROKER_ID = "HBOT-" + + +# deeply merge two dictionaries +def merge_dicts(source: Dict, destination: Dict) -> Dict: + for key, value in source.items(): + if isinstance(value, dict): + # get node or create one + node = destination.setdefault(key, {}) + merge_dicts(value, node) + else: + destination[key] = value + + return destination + + +# join paths +def join_paths(*paths: List[str]) -> str: + return "/".join(paths) + + +# get timestamp in milliseconds +def get_ms_timestamp() -> int: + return get_tracking_nonce_low_res() + + +# convert milliseconds timestamp to seconds +def ms_timestamp_to_s(ms: int) -> int: + return math.floor(ms / 1e3) + + +# Request ID class +class RequestId: + """ + Generate request ids + """ + _request_id: int = 0 + + @classmethod + def generate_request_id(cls) -> int: + return get_tracking_nonce() + + +def convert_from_exchange_trading_pair(exchange_trading_pair: str) -> str: + return exchange_trading_pair.replace("_", "-").upper() + + +def convert_from_ws_trading_pair(exchange_trading_pair: str) -> str: + return exchange_trading_pair.replace("_", "-") + + +def convert_to_exchange_trading_pair(hb_trading_pair: str) -> str: + return hb_trading_pair.replace("-", "_").lower() + + +def convert_to_ws_trading_pair(hb_trading_pair: str) -> str: + return hb_trading_pair.replace("-", "_") + + +def get_new_client_order_id(is_buy: bool, trading_pair: str) -> str: + side = "B" if is_buy else "S" + return f"{HBOT_BROKER_ID}{side}-{trading_pair}-{get_tracking_nonce()}" + + +def get_api_reason(code: str) -> str: + return Constants.API_REASONS.get(int(code), code) + + +KEYS = { + "digifinex_api_key": + ConfigVar(key="digifinex_api_key", + prompt="Enter your Digifinex API key >>> ", + required_if=using_exchange("digifinex"), + is_secure=True, + is_connect_key=True), + "digifinex_secret_key": + ConfigVar(key="digifinex_secret_key", + prompt="Enter your Digifinex secret key >>> ", + required_if=using_exchange("digifinex"), + is_secure=True, + is_connect_key=True), +} diff --git a/hummingbot/connector/exchange/digifinex/digifinex_websocket.py b/hummingbot/connector/exchange/digifinex/digifinex_websocket.py new file mode 100644 index 0000000000..c2ca969778 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/digifinex_websocket.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python +import asyncio +import copy +import logging +import websockets +import zlib +import ujson +from asyncio import InvalidStateError +import hummingbot.connector.exchange.digifinex.digifinex_constants as constants +# from hummingbot.core.utils.async_utils import safe_ensure_future + + +from typing import Optional, AsyncIterable, Any, List +from websockets.exceptions import ConnectionClosed +from hummingbot.logger import HummingbotLogger +from hummingbot.connector.exchange.digifinex.digifinex_auth import DigifinexAuth +from hummingbot.connector.exchange.digifinex.digifinex_utils import RequestId + +# reusable websocket class +# ToDo: We should eventually remove this class, and instantiate web socket connection normally (see Binance for example) + + +class DigifinexWebsocket(RequestId): + MESSAGE_TIMEOUT = 30.0 + PING_TIMEOUT = 10.0 + _logger: Optional[HummingbotLogger] = None + disconnect_future: asyncio.Future = None + tasks: [asyncio.Task] = [] + login_msg_id: int = 0 + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, auth: Optional[DigifinexAuth] = None): + self._auth: Optional[DigifinexAuth] = auth + self._isPrivate = True if self._auth is not None else False + self._WS_URL = constants.WSS_PRIVATE_URL if self._isPrivate else constants.WSS_PUBLIC_URL + self._client: Optional[websockets.WebSocketClientProtocol] = None + + # connect to exchange + async def connect(self): + if self.disconnect_future is not None: + raise InvalidStateError('already connected') + self.disconnect_future = asyncio.Future() + + try: + self._client = await websockets.connect(self._WS_URL) + + # if auth class was passed into websocket class + # we need to emit authenticated requests + if self._isPrivate: + await self.login() + self.tasks.append(asyncio.create_task(self._ping_loop())) + + return self._client + except Exception as e: + self.logger().error(f"Websocket error: '{str(e)}'", exc_info=True) + + async def login(self): + self.login_msg_id = await self._emit("server.auth", self._auth.generate_ws_signature()) + msg = await self._messages() + if msg is None: + raise ConnectionError('websocket auth failed: connection closed unexpectedly') + if msg.get('error') is not None: + raise ConnectionError(f'websocket auth failed: {msg}') + + # disconnect from exchange + async def disconnect(self): + if self._client is None: + return + + await self._client.close() + if not self.disconnect_future.done: + self.disconnect_future.result(True) + if len(self.tasks) > 0: + await asyncio.wait(self.tasks) + + async def _ping_loop(self): + while True: + try: + disconnected = await asyncio.wait_for(self.disconnect_future, 30) + _ = disconnected + break + except asyncio.TimeoutError: + await self._emit('server.ping', []) + # msg = await self._messages() # concurrent read not allowed + + # receive & parse messages + async def _messages(self) -> Any: + try: + success = False + while True: + try: + raw_msg_bytes: bytes = await asyncio.wait_for(self._client.recv(), timeout=self.MESSAGE_TIMEOUT) + inflated_msg: bytes = zlib.decompress(raw_msg_bytes) + raw_msg = ujson.loads(inflated_msg) + # if "method" in raw_msg and raw_msg["method"] == "server.ping": + # payload = {"id": raw_msg["id"], "method": "public/respond-heartbeat"} + # safe_ensure_future(self._client.send(ujson.dumps(payload))) + # self.logger().debug(inflated_msg) + # method = raw_msg.get('method') + # if method not in ['depth.update', 'trades.update']: + # self.logger().network(inflated_msg) + + err = raw_msg.get('error') + if err is not None: + raise ConnectionError(raw_msg) + elif raw_msg.get('result') == 'pong': + continue # ignore ping response + + success = True + return raw_msg + except asyncio.TimeoutError: + await asyncio.wait_for(self._client.ping(), timeout=self.PING_TIMEOUT) + except asyncio.TimeoutError: + self.logger().warning("WebSocket ping timed out. Going to reconnect...") + return + except ConnectionClosed: + return + except Exception as e: + _ = e + self.logger().exception('digifinex.websocket._messages', stack_info=True) + raise + finally: + if not success: + await self.disconnect() + + # emit messages + async def _emit(self, method: str, data: Optional[Any] = {}) -> int: + id = self.generate_request_id() + + payload = { + "id": id, + "method": method, + "params": copy.deepcopy(data), + } + + req = ujson.dumps(payload) + self.logger().network(req) # todo remove log + await self._client.send(req) + + return id + + # request via websocket + async def request(self, method: str, data: Optional[Any] = {}) -> int: + return await self._emit(method, data) + + # subscribe to a method + async def subscribe(self, category: str, channels: List[str]) -> int: + id = await self.request(category + ".subscribe", channels) + msg = await self._messages() + if msg.get('error') is not None: + raise ConnectionError(f'subscribe {category} {channels} failed: {msg}') + return id + + # unsubscribe to a method + async def unsubscribe(self, channels: List[str]) -> int: + return await self.request("unsubscribe", { + "channels": channels + }) + + # listen to messages by method + async def on_message(self) -> AsyncIterable[Any]: + while True: + msg = await self._messages() + if msg is None: + return + if 'pong' in str(msg): + _ = int(0) + yield msg diff --git a/hummingbot/connector/exchange/digifinex/time_patcher.py b/hummingbot/connector/exchange/digifinex/time_patcher.py new file mode 100644 index 0000000000..3c6d195f30 --- /dev/null +++ b/hummingbot/connector/exchange/digifinex/time_patcher.py @@ -0,0 +1,112 @@ +import asyncio +from collections import deque +import logging +import statistics +import time +from typing import Deque, Optional, Callable, Awaitable + +from hummingbot.logger import HummingbotLogger +from hummingbot.core.utils.async_utils import safe_ensure_future + + +class TimePatcher: + # BINANCE_TIME_API = "https://api.binance.com/api/v1/time" + NaN = float("nan") + _bt_logger = None + _bt_shared_instance = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._bt_logger is None: + cls._bt_logger = logging.getLogger(__name__) + return cls._bt_logger + + # query_time_func returns the server time in seconds + def __init__(self, exchange_name: str, query_time_func: Callable[[], Awaitable[float]], check_interval: float = 60.0): + self._exchange_name = exchange_name + self._query_time_func = query_time_func + self._time_offset_ms: Deque[float] = deque([]) + self._set_server_time_offset_task: Optional[asyncio.Task] = None + self._started: bool = False + self._server_time_offset_check_interval = check_interval + self._median_window = 5 + self._last_update_local_time: float = self.NaN + self._scheduled_update_task: Optional[asyncio.Task] = None + + @property + def started(self) -> bool: + return self._started + + @property + def time_offset_ms(self) -> float: + if not self._time_offset_ms: + return (time.time() - time.perf_counter()) * 1e3 + return statistics.median(self._time_offset_ms) + + def add_time_offset_ms_sample(self, offset: float): + self._time_offset_ms.append(offset) + while len(self._time_offset_ms) > self._median_window: + self._time_offset_ms.popleft() + + def clear_time_offset_ms_samples(self): + self._time_offset_ms.clear() + + def time(self) -> float: + return time.perf_counter() + self.time_offset_ms * 1e-3 + + def start(self): + if self._set_server_time_offset_task is None: + self._set_server_time_offset_task = safe_ensure_future(self.update_server_time_offset_loop()) + self._started = True + + def stop(self): + if self._set_server_time_offset_task: + self._set_server_time_offset_task.cancel() + self._set_server_time_offset_task = None + self._time_offset_ms.clear() + self._started = False + + def schedule_update_server_time_offset(self) -> asyncio.Task: + # If an update task is already scheduled, don't do anything. + if self._scheduled_update_task is not None and not self._scheduled_update_task.done(): + return self._scheduled_update_task + + current_local_time: float = time.perf_counter() + if not (current_local_time - self._last_update_local_time < 5): + # If there was no recent update, schedule the server time offset update immediately. + self._scheduled_update_task = safe_ensure_future(self.update_server_time_offset()) + else: + # If there was a recent update, schedule the server time offset update after 5 seconds. + async def update_later(): + await asyncio.sleep(5.0) + await self.update_server_time_offset() + self._scheduled_update_task = safe_ensure_future(update_later()) + + return self._scheduled_update_task + + async def update_server_time_offset_loop(self): + while True: + await self.update_server_time_offset() + await asyncio.sleep(self._server_time_offset_check_interval) + + async def update_server_time_offset(self): + try: + local_before_ms: float = time.perf_counter() * 1e3 + query_time_func = self._query_time_func.__func__ + server_time = await query_time_func() + # async with aiohttp.ClientSession() as session: + # async with session.get(self.BINANCE_TIME_API) as resp: + # resp_data: Dict[str, float] = await resp.json() + # binance_server_time_ms: float = float(resp_data["serverTime"]) + # local_after_ms: float = time.perf_counter() * 1e3 + local_after_ms: float = time.perf_counter() * 1e3 + local_server_time_pre_image_ms: float = (local_before_ms + local_after_ms) / 2.0 + time_offset_ms: float = server_time * 1000 - local_server_time_pre_image_ms + self.add_time_offset_ms_sample(time_offset_ms) + self._last_update_local_time = time.perf_counter() + except asyncio.CancelledError: + raise + except Exception: + self.logger().network(f"Error getting {self._exchange_name} server time.", exc_info=True, + app_warning_msg=f"Could not refresh {self._exchange_name} server time. " + "Check network connection.") diff --git a/hummingbot/connector/exchange/dydx/dydx_api_order_book_data_source.py b/hummingbot/connector/exchange/dydx/dydx_api_order_book_data_source.py index 8eab204355..447f3863af 100644 --- a/hummingbot/connector/exchange/dydx/dydx_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/dydx/dydx_api_order_book_data_source.py @@ -1,19 +1,10 @@ #!/usr/bin/env python import asyncio -from decimal import Decimal from datetime import datetime - import aiohttp import logging -# import pandas as pd -# import math - -import requests -import cachetools.func - from typing import AsyncIterable, Dict, List, Optional, Any - import time import ujson import websockets @@ -22,7 +13,7 @@ from hummingbot.connector.exchange.dydx.dydx_order_book import DydxOrderBook from hummingbot.connector.exchange.dydx.dydx_active_order_tracker import DydxActiveOrderTracker from hummingbot.connector.exchange.dydx.dydx_api_token_configuration_data_source import DydxAPITokenConfigurationDataSource -from hummingbot.connector.exchange.dydx.dydx_utils import convert_from_exchange_trading_pair, convert_to_exchange_trading_pair, convert_v2_pair_to_v1 +from hummingbot.connector.exchange.dydx.dydx_utils import convert_from_exchange_trading_pair, convert_v2_pair_to_v1 from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource from hummingbot.logger import HummingbotLogger from hummingbot.core.data_type.order_book import OrderBook @@ -139,22 +130,6 @@ async def _inner_messages(self, ws: websockets.WebSocketClientProtocol) -> Async finally: await ws.close() - @staticmethod - @cachetools.func.ttl_cache(ttl=10) - def get_mid_price(trading_pair: str) -> Optional[Decimal]: - exchange_pair: str = convert_to_exchange_trading_pair(trading_pair) - market_info_response = requests.get(url=DYDX_MARKET_INFO_URL.format(exchange_pair)) - market_info = market_info_response.json() - base_decimals = market_info['market']['baseCurrency']['decimals'] - quote_decimals = market_info['market']['quoteCurrency']['decimals'] - - resp = requests.get(url=DYDX_ORDERBOOK_URL.format(exchange_pair)) - record = resp.json() - conversion_factor = Decimal(f"1e{base_decimals - quote_decimals}") - best_bid = Decimal(record["bids"][0]["price"]) * conversion_factor - best_ask = Decimal(record["asks"][0]["price"]) * conversion_factor - return (best_bid + best_ask) / 2 - @staticmethod async def fetch_trading_pairs() -> List[str]: try: diff --git a/hummingbot/connector/exchange/hitbtc/__init__.py b/hummingbot/connector/exchange/hitbtc/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pxd b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pxd new file mode 100644 index 0000000000..5babac5332 --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pxd @@ -0,0 +1,11 @@ +# distutils: language=c++ +cimport numpy as np + +cdef class HitbtcActiveOrderTracker: + cdef dict _active_bids + cdef dict _active_asks + + cdef tuple c_convert_diff_message_to_np_arrays(self, object message) + cdef tuple c_convert_snapshot_message_to_np_arrays(self, object message) + # This method doesn't seem to be used anywhere at all + # cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx new file mode 100644 index 0000000000..5e248bb3d5 --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx @@ -0,0 +1,155 @@ +# distutils: language=c++ +# distutils: sources=hummingbot/core/cpp/OrderBookEntry.cpp +import logging +import numpy as np +from decimal import Decimal +from typing import Dict +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.order_book_row import OrderBookRow + +_logger = None +s_empty_diff = np.ndarray(shape=(0, 4), dtype="float64") +HitbtcOrderBookTrackingDictionary = Dict[Decimal, Dict[str, Dict[str, any]]] + +cdef class HitbtcActiveOrderTracker: + def __init__(self, + active_asks: HitbtcOrderBookTrackingDictionary = None, + active_bids: HitbtcOrderBookTrackingDictionary = None): + super().__init__() + self._active_asks = active_asks or {} + self._active_bids = active_bids or {} + + @classmethod + def logger(cls) -> HummingbotLogger: + global _logger + if _logger is None: + _logger = logging.getLogger(__name__) + return _logger + + @property + def active_asks(self) -> HitbtcOrderBookTrackingDictionary: + return self._active_asks + + @property + def active_bids(self) -> HitbtcOrderBookTrackingDictionary: + return self._active_bids + + # TODO: research this more + def volume_for_ask_price(self, price) -> float: + return NotImplementedError + + # TODO: research this more + def volume_for_bid_price(self, price) -> float: + return NotImplementedError + + def get_rates_and_quantities(self, entry) -> tuple: + # price, quantity + return float(entry["price"]), float(entry["size"]) + + cdef tuple c_convert_diff_message_to_np_arrays(self, object message): + cdef: + dict content = message.content + list content_keys = list(content.keys()) + list bid_entries = [] + list ask_entries = [] + str order_id + str order_side + str price_raw + object price + dict order_dict + double timestamp = message.timestamp + double amount = 0 + + if "bid" in content_keys: + bid_entries = content["bid"] + if "ask" in content_keys: + ask_entries = content["ask"] + + bids = s_empty_diff + asks = s_empty_diff + + if len(bid_entries) > 0: + bids = np.array( + [[timestamp, + price, + amount, + message.update_id] + for price, amount in [self.get_rates_and_quantities(entry) for entry in bid_entries]], + dtype="float64", + ndmin=2 + ) + + if len(ask_entries) > 0: + asks = np.array( + [[timestamp, + price, + amount, + message.update_id] + for price, amount in [self.get_rates_and_quantities(entry) for entry in ask_entries]], + dtype="float64", + ndmin=2 + ) + + return bids, asks + + cdef tuple c_convert_snapshot_message_to_np_arrays(self, object message): + cdef: + float price + float amount + str order_id + dict order_dict + + # Refresh all order tracking. + self._active_bids.clear() + self._active_asks.clear() + timestamp = message.timestamp + content = message.content + + for snapshot_orders, active_orders in [(content["bid"], self._active_bids), (content["ask"], self._active_asks)]: + for entry in snapshot_orders: + price, amount = self.get_rates_and_quantities(entry) + active_orders[price] = amount + + # Return the sorted snapshot tables. + cdef: + np.ndarray[np.float64_t, ndim=2] bids = np.array( + [[message.timestamp, + float(price), + float(self._active_bids[price]), + message.update_id] + for price in sorted(self._active_bids.keys())], dtype='float64', ndmin=2) + np.ndarray[np.float64_t, ndim=2] asks = np.array( + [[message.timestamp, + float(price), + float(self._active_asks[price]), + message.update_id] + for price in sorted(self._active_asks.keys(), reverse=True)], dtype='float64', ndmin=2) + + if bids.shape[1] != 4: + bids = bids.reshape((0, 4)) + if asks.shape[1] != 4: + asks = asks.reshape((0, 4)) + + return bids, asks + + # This method doesn't seem to be used anywhere at all + # cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message): + # cdef: + # double trade_type_value = 1.0 if message.content["side"] == "buy" else 2.0 + # list content = message.content + # return np.array( + # [message.timestamp, trade_type_value, float(content["price"]), float(content["quantity"])], + # dtype="float64" + # ) + + def convert_diff_message_to_order_book_row(self, message): + np_bids, np_asks = self.c_convert_diff_message_to_np_arrays(message) + bids_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_bids] + asks_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_asks] + return bids_row, asks_row + + def convert_snapshot_message_to_order_book_row(self, message): + np_bids, np_asks = self.c_convert_snapshot_message_to_np_arrays(message) + bids_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_bids] + asks_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_asks] + return bids_row, asks_row diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py new file mode 100644 index 0000000000..40d83516da --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python +import asyncio +import logging +import time +import pandas as pd +from decimal import Decimal +from typing import Optional, List, Dict, Any +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_message import OrderBookMessage +from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource +from hummingbot.logger import HummingbotLogger +from .hitbtc_constants import Constants +from .hitbtc_active_order_tracker import HitbtcActiveOrderTracker +from .hitbtc_order_book import HitbtcOrderBook +from .hitbtc_websocket import HitbtcWebsocket +from .hitbtc_utils import ( + str_date_to_ts, + convert_to_exchange_trading_pair, + convert_from_exchange_trading_pair, + api_call_with_retries, + HitbtcAPIError, +) + + +class HitbtcAPIOrderBookDataSource(OrderBookTrackerDataSource): + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, trading_pairs: List[str] = None): + super().__init__(trading_pairs) + self._trading_pairs: List[str] = trading_pairs + self._snapshot_msg: Dict[str, any] = {} + + @classmethod + async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, Decimal]: + results = {} + if len(trading_pairs) > 1: + tickers: List[Dict[Any]] = await api_call_with_retries("GET", Constants.ENDPOINT["TICKER"]) + for trading_pair in trading_pairs: + ex_pair: str = convert_to_exchange_trading_pair(trading_pair) + if len(trading_pairs) > 1: + ticker: Dict[Any] = list([tic for tic in tickers if tic['symbol'] == ex_pair])[0] + else: + url_endpoint = Constants.ENDPOINT["TICKER_SINGLE"].format(trading_pair=ex_pair) + ticker: Dict[Any] = await api_call_with_retries("GET", url_endpoint) + results[trading_pair]: Decimal = Decimal(str(ticker["last"])) + return results + + @staticmethod + async def fetch_trading_pairs() -> List[str]: + try: + symbols: List[Dict[str, Any]] = await api_call_with_retries("GET", Constants.ENDPOINT["SYMBOL"]) + trading_pairs: List[str] = list([convert_from_exchange_trading_pair(sym["id"]) for sym in symbols]) + # Filter out unmatched pairs so nothing breaks + return [sym for sym in trading_pairs if sym is not None] + except Exception: + # Do nothing if the request fails -- there will be no autocomplete for HitBTC trading pairs + pass + return [] + + @staticmethod + async def get_order_book_data(trading_pair: str) -> Dict[str, any]: + """ + Get whole orderbook + """ + try: + ex_pair = convert_to_exchange_trading_pair(trading_pair) + orderbook_response: Dict[Any] = await api_call_with_retries("GET", Constants.ENDPOINT["ORDER_BOOK"], + params={"limit": 150, "symbols": ex_pair}) + return orderbook_response[ex_pair] + except HitbtcAPIError as e: + err = e.error_payload.get('error', e.error_payload) + raise IOError( + f"Error fetching OrderBook for {trading_pair} at {Constants.EXCHANGE_NAME}. " + f"HTTP status is {e.error_payload['status']}. Error is {err.get('message', str(err))}.") + + async def get_new_order_book(self, trading_pair: str) -> OrderBook: + snapshot: Dict[str, Any] = await self.get_order_book_data(trading_pair) + snapshot_timestamp: float = time.time() + snapshot_msg: OrderBookMessage = HitbtcOrderBook.snapshot_message_from_exchange( + snapshot, + snapshot_timestamp, + metadata={"trading_pair": trading_pair}) + order_book = self.order_book_create_function() + active_order_tracker: HitbtcActiveOrderTracker = HitbtcActiveOrderTracker() + bids, asks = active_order_tracker.convert_snapshot_message_to_order_book_row(snapshot_msg) + order_book.apply_snapshot(bids, asks, snapshot_msg.update_id) + return order_book + + async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for trades using websocket trade channel + """ + while True: + try: + ws = HitbtcWebsocket() + await ws.connect() + + for pair in self._trading_pairs: + await ws.subscribe(Constants.WS_SUB["TRADES"], convert_to_exchange_trading_pair(pair)) + + async for response in ws.on_message(): + method: str = response.get("method", None) + trades_data: str = response.get("params", None) + + if trades_data is None or method != Constants.WS_METHODS['TRADES_UPDATE']: + continue + + pair: str = convert_from_exchange_trading_pair(response["params"]["symbol"]) + + for trade in trades_data["data"]: + trade: Dict[Any] = trade + trade_timestamp: int = str_date_to_ts(trade["timestamp"]) + trade_msg: OrderBookMessage = HitbtcOrderBook.trade_message_from_exchange( + trade, + trade_timestamp, + metadata={"trading_pair": pair}) + output.put_nowait(trade_msg) + + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error.", exc_info=True) + await asyncio.sleep(5.0) + finally: + await ws.disconnect() + + async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for orderbook diffs using websocket book channel + """ + while True: + try: + ws = HitbtcWebsocket() + await ws.connect() + + order_book_methods = [ + Constants.WS_METHODS['ORDERS_SNAPSHOT'], + Constants.WS_METHODS['ORDERS_UPDATE'], + ] + + for pair in self._trading_pairs: + await ws.subscribe(Constants.WS_SUB["ORDERS"], convert_to_exchange_trading_pair(pair)) + + async for response in ws.on_message(): + method: str = response.get("method", None) + order_book_data: str = response.get("params", None) + + if order_book_data is None or method not in order_book_methods: + continue + + timestamp: int = str_date_to_ts(order_book_data["timestamp"]) + pair: str = convert_from_exchange_trading_pair(order_book_data["symbol"]) + + order_book_msg_cls = (HitbtcOrderBook.diff_message_from_exchange + if method == Constants.WS_METHODS['ORDERS_UPDATE'] else + HitbtcOrderBook.snapshot_message_from_exchange) + + orderbook_msg: OrderBookMessage = order_book_msg_cls( + order_book_data, + timestamp, + metadata={"trading_pair": pair}) + output.put_nowait(orderbook_msg) + + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unexpected error with WebSocket connection.", exc_info=True, + app_warning_msg="Unexpected error with WebSocket connection. Retrying in 30 seconds. " + "Check network connection.") + await asyncio.sleep(30.0) + finally: + await ws.disconnect() + + async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for orderbook snapshots by fetching orderbook + """ + while True: + try: + for trading_pair in self._trading_pairs: + try: + snapshot: Dict[str, any] = await self.get_order_book_data(trading_pair) + snapshot_timestamp: int = str_date_to_ts(snapshot["timestamp"]) + snapshot_msg: OrderBookMessage = HitbtcOrderBook.snapshot_message_from_exchange( + snapshot, + snapshot_timestamp, + metadata={"trading_pair": trading_pair} + ) + output.put_nowait(snapshot_msg) + self.logger().debug(f"Saved order book snapshot for {trading_pair}") + # Be careful not to go above API rate limits. + await asyncio.sleep(5.0) + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unexpected error with WebSocket connection.", exc_info=True, + app_warning_msg="Unexpected error with WebSocket connection. Retrying in 5 seconds. " + "Check network connection.") + await asyncio.sleep(5.0) + this_hour: pd.Timestamp = pd.Timestamp.utcnow().replace(minute=0, second=0, microsecond=0) + next_hour: pd.Timestamp = this_hour + pd.Timedelta(hours=1) + delta: float = next_hour.timestamp() - time.time() + await asyncio.sleep(delta) + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error.", exc_info=True) + await asyncio.sleep(5.0) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py new file mode 100755 index 0000000000..954ab9c344 --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +import time +import asyncio +import logging +from typing import ( + Any, + AsyncIterable, + List, + Optional, +) +from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource +from hummingbot.logger import HummingbotLogger +from .hitbtc_constants import Constants +from .hitbtc_auth import HitbtcAuth +from .hitbtc_utils import HitbtcAPIError +from .hitbtc_websocket import HitbtcWebsocket + + +class HitbtcAPIUserStreamDataSource(UserStreamTrackerDataSource): + + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, hitbtc_auth: HitbtcAuth, trading_pairs: Optional[List[str]] = []): + self._hitbtc_auth: HitbtcAuth = hitbtc_auth + self._ws: HitbtcWebsocket = None + self._trading_pairs = trading_pairs + self._current_listen_key = None + self._listen_for_user_stream_task = None + self._last_recv_time: float = 0 + super().__init__() + + @property + def last_recv_time(self) -> float: + return self._last_recv_time + + async def _ws_request_balances(self): + return await self._ws.request(Constants.WS_METHODS["USER_BALANCE"]) + + async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: + """ + Subscribe to active orders via web socket + """ + + try: + self._ws = HitbtcWebsocket(self._hitbtc_auth) + + await self._ws.connect() + + await self._ws.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"], None, {}) + + event_methods = [ + Constants.WS_METHODS["USER_ORDERS"], + Constants.WS_METHODS["USER_TRADES"], + ] + + async for msg in self._ws.on_message(): + self._last_recv_time = time.time() + + if msg.get("params", msg.get("result", None)) is None: + continue + elif msg.get("method", None) in event_methods: + await self._ws_request_balances() + yield msg + except Exception as e: + raise e + finally: + await self._ws.disconnect() + await asyncio.sleep(5) + + async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue) -> AsyncIterable[Any]: + """ + *required + Subscribe to user stream via web socket, and keep the connection open for incoming messages + :param ev_loop: ev_loop to execute this function in + :param output: an async queue where the incoming messages are stored + """ + + while True: + try: + async for msg in self._listen_to_orders_trades_balances(): + output.put_nowait(msg) + except asyncio.CancelledError: + raise + except HitbtcAPIError as e: + self.logger().error(e.error_payload.get('error'), exc_info=True) + raise + except Exception: + self.logger().error( + f"Unexpected error with {Constants.EXCHANGE_NAME} WebSocket connection. " + "Retrying after 30 seconds...", exc_info=True) + await asyncio.sleep(30.0) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py b/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py new file mode 100755 index 0000000000..be37f2e149 --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py @@ -0,0 +1,72 @@ +import hmac +import hashlib +import time +from base64 import b64encode +from typing import Dict, Any + + +class HitbtcAuth(): + """ + Auth class required by HitBTC API + Learn more at https://exchange-docs.crypto.com/#digital-signature + """ + def __init__(self, api_key: str, secret_key: str): + self.api_key = api_key + self.secret_key = secret_key + + def generate_payload( + self, + method: str, + url: str, + params: Dict[str, Any] = None, + ): + """ + Generates authentication payload and returns it. + :return: A base64 encoded payload for the authentication header. + """ + # Nonce is standard EPOCH timestamp only accurate to 1s + nonce = str(int(time.time())) + body = "" + # Need to build the full URL with query string for HS256 sig + if params is not None and len(params) > 0: + query_string = "&".join([f"{k}={v}" for k, v in params.items()]) + if method == "GET": + url = f"{url}?{query_string}" + else: + body = query_string + # Concat payload + payload = f"{method}{nonce}{url}{body}" + # Create HS256 sig + sig = hmac.new(self.secret_key.encode(), payload.encode(), hashlib.sha256).hexdigest() + # Base64 encode it with public key and nonce + return b64encode(f"{self.api_key}:{nonce}:{sig}".encode()).decode().strip() + + def generate_auth_dict_ws(self, + nonce: int): + """ + Generates an authentication params for HitBTC websockets login + :return: a dictionary of auth params + """ + return { + "algo": "HS256", + "pKey": str(self.api_key), + "nonce": str(nonce), + "signature": hmac.new(self.secret_key.encode('utf-8'), + str(nonce).encode('utf-8'), + hashlib.sha256).hexdigest() + } + + def get_headers(self, + method, + url, + params) -> Dict[str, Any]: + """ + Generates authentication headers required by HitBTC + :return: a dictionary of auth headers + """ + payload = self.generate_payload(method, url, params) + headers = { + "Authorization": f"HS256 {payload}", + "Content-Type": "application/x-www-form-urlencoded", + } + return headers diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py new file mode 100644 index 0000000000..538e0b21f2 --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py @@ -0,0 +1,57 @@ +# A single source of truth for constant variables related to the exchange +class Constants: + EXCHANGE_NAME = "hitbtc" + REST_URL = "https://api.hitbtc.com/api/2" + REST_URL_AUTH = "/api/2" + WS_PRIVATE_URL = "wss://api.hitbtc.com/api/2/ws/trading" + WS_PUBLIC_URL = "wss://api.hitbtc.com/api/2/ws/public" + + HBOT_BROKER_ID = "refzzz48" + + ENDPOINT = { + # Public Endpoints + "TICKER": "public/ticker", + "TICKER_SINGLE": "public/ticker/{trading_pair}", + "SYMBOL": "public/symbol", + "ORDER_BOOK": "public/orderbook", + "ORDER_CREATE": "order", + "ORDER_DELETE": "order/{id}", + "ORDER_STATUS": "order/{id}", + "USER_ORDERS": "order", + "USER_BALANCES": "trading/balance", + } + + WS_SUB = { + "TRADES": "Trades", + "ORDERS": "Orderbook", + "USER_ORDERS_TRADES": "Reports", + + } + + WS_METHODS = { + "ORDERS_SNAPSHOT": "snapshotOrderbook", + "ORDERS_UPDATE": "updateOrderbook", + "TRADES_SNAPSHOT": "snapshotTrades", + "TRADES_UPDATE": "updateTrades", + "USER_BALANCE": "getTradingBalance", + "USER_ORDERS": "activeOrders", + "USER_TRADES": "report", + } + + # Timeouts + MESSAGE_TIMEOUT = 30.0 + PING_TIMEOUT = 10.0 + API_CALL_TIMEOUT = 10.0 + API_MAX_RETRIES = 4 + + # Intervals + # Only used when nothing is received from WS + SHORT_POLL_INTERVAL = 5.0 + # One minute should be fine since we get trades, orders and balances via WS + LONG_POLL_INTERVAL = 60.0 + UPDATE_ORDER_STATUS_INTERVAL = 60.0 + # 10 minute interval to update trading rules, these would likely never change whilst running. + INTERVAL_TRADING_RULES = 600 + + # Trading pair splitter regex + TRADING_PAIR_SPLITTER = r"^(\w+)(BTC|BCH|DAI|DDRST|EOSDT|EOS|ETH|EURS|IDRT|PAX|BUSD|GUSD|TUSD|USDC|USDT|USD)$" diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py new file mode 100644 index 0000000000..9f6f83ec15 --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -0,0 +1,877 @@ +import logging +from typing import ( + Dict, + List, + Optional, + Any, + AsyncIterable, +) +from decimal import Decimal +import asyncio +import aiohttp +import math +import time +from async_timeout import timeout + +from hummingbot.core.network_iterator import NetworkStatus +from hummingbot.logger import HummingbotLogger +from hummingbot.core.clock import Clock +from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather +from hummingbot.connector.trading_rule import TradingRule +from hummingbot.core.data_type.cancellation_result import CancellationResult +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.limit_order import LimitOrder +from hummingbot.core.event.events import ( + MarketEvent, + BuyOrderCompletedEvent, + SellOrderCompletedEvent, + OrderFilledEvent, + OrderCancelledEvent, + BuyOrderCreatedEvent, + SellOrderCreatedEvent, + MarketOrderFailureEvent, + OrderType, + TradeType, + TradeFee +) +from hummingbot.connector.exchange_base import ExchangeBase +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_tracker import HitbtcOrderBookTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_user_stream_tracker import HitbtcUserStreamTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth +from hummingbot.connector.exchange.hitbtc.hitbtc_in_flight_order import HitbtcInFlightOrder +from hummingbot.connector.exchange.hitbtc.hitbtc_utils import ( + convert_from_exchange_trading_pair, + convert_to_exchange_trading_pair, + get_new_client_order_id, + aiohttp_response_with_errors, + retry_sleep_time, + str_date_to_ts, + HitbtcAPIError, +) +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants +from hummingbot.core.data_type.common import OpenOrder +ctce_logger = None +s_decimal_NaN = Decimal("nan") + + +class HitbtcExchange(ExchangeBase): + """ + HitbtcExchange connects with HitBTC exchange and provides order book pricing, user account tracking and + trading functionality. + """ + ORDER_NOT_EXIST_CONFIRMATION_COUNT = 3 + ORDER_NOT_EXIST_CANCEL_COUNT = 2 + + @classmethod + def logger(cls) -> HummingbotLogger: + global ctce_logger + if ctce_logger is None: + ctce_logger = logging.getLogger(__name__) + return ctce_logger + + def __init__(self, + hitbtc_api_key: str, + hitbtc_secret_key: str, + trading_pairs: Optional[List[str]] = None, + trading_required: bool = True + ): + """ + :param hitbtc_api_key: The API key to connect to private HitBTC APIs. + :param hitbtc_secret_key: The API secret. + :param trading_pairs: The market trading pairs which to track order book data. + :param trading_required: Whether actual trading is needed. + """ + super().__init__() + self._trading_required = trading_required + self._trading_pairs = trading_pairs + self._hitbtc_auth = HitbtcAuth(hitbtc_api_key, hitbtc_secret_key) + self._order_book_tracker = HitbtcOrderBookTracker(trading_pairs=trading_pairs) + self._user_stream_tracker = HitbtcUserStreamTracker(self._hitbtc_auth, trading_pairs) + self._ev_loop = asyncio.get_event_loop() + self._shared_client = None + self._poll_notifier = asyncio.Event() + self._last_timestamp = 0 + self._in_flight_orders = {} # Dict[client_order_id:str, HitbtcInFlightOrder] + self._order_not_found_records = {} # Dict[client_order_id:str, count:int] + self._trading_rules = {} # Dict[trading_pair:str, TradingRule] + self._status_polling_task = None + self._user_stream_event_listener_task = None + self._trading_rules_polling_task = None + self._last_poll_timestamp = 0 + + @property + def name(self) -> str: + return "hitbtc" + + @property + def order_books(self) -> Dict[str, OrderBook]: + return self._order_book_tracker.order_books + + @property + def trading_rules(self) -> Dict[str, TradingRule]: + return self._trading_rules + + @property + def in_flight_orders(self) -> Dict[str, HitbtcInFlightOrder]: + return self._in_flight_orders + + @property + def status_dict(self) -> Dict[str, bool]: + """ + A dictionary of statuses of various connector's components. + """ + return { + "order_books_initialized": self._order_book_tracker.ready, + "account_balance": len(self._account_balances) > 0 if self._trading_required else True, + "trading_rule_initialized": len(self._trading_rules) > 0, + "user_stream_initialized": + self._user_stream_tracker.data_source.last_recv_time > 0 if self._trading_required else True, + } + + @property + def ready(self) -> bool: + """ + :return True when all statuses pass, this might take 5-10 seconds for all the connector's components and + services to be ready. + """ + return all(self.status_dict.values()) + + @property + def limit_orders(self) -> List[LimitOrder]: + return [ + in_flight_order.to_limit_order() + for in_flight_order in self._in_flight_orders.values() + ] + + @property + def tracking_states(self) -> Dict[str, any]: + """ + :return active in-flight orders in json format, is used to save in sqlite db. + """ + return { + key: value.to_json() + for key, value in self._in_flight_orders.items() + if not value.is_done + } + + def restore_tracking_states(self, saved_states: Dict[str, any]): + """ + Restore in-flight orders from saved tracking states, this is st the connector can pick up on where it left off + when it disconnects. + :param saved_states: The saved tracking_states. + """ + self._in_flight_orders.update({ + key: HitbtcInFlightOrder.from_json(value) + for key, value in saved_states.items() + }) + + def supported_order_types(self) -> List[OrderType]: + """ + :return a list of OrderType supported by this connector. + Note that Market order type is no longer required and will not be used. + """ + return [OrderType.LIMIT, OrderType.LIMIT_MAKER] + + def start(self, clock: Clock, timestamp: float): + """ + This function is called automatically by the clock. + """ + super().start(clock, timestamp) + + def stop(self, clock: Clock): + """ + This function is called automatically by the clock. + """ + super().stop(clock) + + async def start_network(self): + """ + This function is required by NetworkIterator base class and is called automatically. + It starts tracking order book, polling trading rules, + updating statuses and tracking user data. + """ + self._order_book_tracker.start() + self._trading_rules_polling_task = safe_ensure_future(self._trading_rules_polling_loop()) + if self._trading_required: + self._status_polling_task = safe_ensure_future(self._status_polling_loop()) + self._user_stream_tracker_task = safe_ensure_future(self._user_stream_tracker.start()) + self._user_stream_event_listener_task = safe_ensure_future(self._user_stream_event_listener()) + + async def stop_network(self): + """ + This function is required by NetworkIterator base class and is called automatically. + """ + self._order_book_tracker.stop() + if self._status_polling_task is not None: + self._status_polling_task.cancel() + self._status_polling_task = None + if self._trading_rules_polling_task is not None: + self._trading_rules_polling_task.cancel() + self._trading_rules_polling_task = None + if self._status_polling_task is not None: + self._status_polling_task.cancel() + self._status_polling_task = None + if self._user_stream_tracker_task is not None: + self._user_stream_tracker_task.cancel() + self._user_stream_tracker_task = None + if self._user_stream_event_listener_task is not None: + self._user_stream_event_listener_task.cancel() + self._user_stream_event_listener_task = None + + async def check_network(self) -> NetworkStatus: + """ + This function is required by NetworkIterator base class and is called periodically to check + the network connection. Simply ping the network (or call any light weight public API). + """ + try: + # since there is no ping endpoint, the lowest rate call is to get BTC-USD symbol + await self._api_request("GET", + Constants.ENDPOINT['SYMBOL'], + params={'symbols': 'BTCUSD'}) + except asyncio.CancelledError: + raise + except Exception: + return NetworkStatus.NOT_CONNECTED + return NetworkStatus.CONNECTED + + async def _http_client(self) -> aiohttp.ClientSession: + """ + :returns Shared client session instance + """ + if self._shared_client is None: + self._shared_client = aiohttp.ClientSession() + return self._shared_client + + async def _trading_rules_polling_loop(self): + """ + Periodically update trading rule. + """ + while True: + try: + await self._update_trading_rules() + await asyncio.sleep(Constants.INTERVAL_TRADING_RULES) + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().network(f"Unexpected error while fetching trading rules. Error: {str(e)}", + exc_info=True, + app_warning_msg=("Could not fetch new trading rules from " + f"{Constants.EXCHANGE_NAME}. Check network connection.")) + await asyncio.sleep(0.5) + + async def _update_trading_rules(self): + symbols_info = await self._api_request("GET", endpoint=Constants.ENDPOINT['SYMBOL']) + self._trading_rules.clear() + self._trading_rules = self._format_trading_rules(symbols_info) + + def _format_trading_rules(self, symbols_info: Dict[str, Any]) -> Dict[str, TradingRule]: + """ + Converts json API response into a dictionary of trading rules. + :param symbols_info: The json API response + :return A dictionary of trading rules. + Response Example: + [ + { + id: "BTCUSD", + baseCurrency: "BTC", + quoteCurrency: "USD", + quantityIncrement: "0.00001", + tickSize: "0.01", + takeLiquidityRate: "0.0025", + provideLiquidityRate: "0.001", + feeCurrency: "USD", + marginTrading: true, + maxInitialLeverage: "12.00" + } + ] + """ + result = {} + for rule in symbols_info: + try: + trading_pair = convert_from_exchange_trading_pair(rule["id"]) + price_step = Decimal(str(rule["tickSize"])) + size_step = Decimal(str(rule["quantityIncrement"])) + result[trading_pair] = TradingRule(trading_pair, + min_order_size=size_step, + min_base_amount_increment=size_step, + min_price_increment=price_step) + except Exception: + self.logger().error(f"Error parsing the trading pair rule {rule}. Skipping.", exc_info=True) + return result + + async def _api_request(self, + method: str, + endpoint: str, + params: Optional[Dict[str, Any]] = None, + is_auth_required: bool = False, + try_count: int = 0) -> Dict[str, Any]: + """ + Sends an aiohttp request and waits for a response. + :param method: The HTTP method, e.g. get or post + :param endpoint: The path url or the API end point + :param params: Additional get/post parameters + :param is_auth_required: Whether an authentication is required, when True the function will add encrypted + signature to the request. + :returns A response in json format. + """ + url = f"{Constants.REST_URL}/{endpoint}" + shared_client = await self._http_client() + # Turn `params` into either GET params or POST body data + qs_params: dict = params if method.upper() == "GET" else None + req_form = aiohttp.FormData(params) if method.upper() == "POST" and params is not None else None + # Generate auth headers if needed. + headers: dict = {"Content-Type": "application/x-www-form-urlencoded"} + if is_auth_required: + headers: dict = self._hitbtc_auth.get_headers(method, f"{Constants.REST_URL_AUTH}/{endpoint}", + params) + # Build request coro + response_coro = shared_client.request(method=method.upper(), url=url, headers=headers, + params=qs_params, data=req_form, + timeout=Constants.API_CALL_TIMEOUT) + http_status, parsed_response, request_errors = await aiohttp_response_with_errors(response_coro) + if request_errors or parsed_response is None: + if try_count < Constants.API_MAX_RETRIES: + try_count += 1 + time_sleep = retry_sleep_time(try_count) + self.logger().info(f"Error fetching data from {url}. HTTP status is {http_status}. " + f"Retrying in {time_sleep:.0f}s.") + await asyncio.sleep(time_sleep) + return await self._api_request(method=method, endpoint=endpoint, params=params, + is_auth_required=is_auth_required, try_count=try_count) + else: + raise HitbtcAPIError({"error": parsed_response, "status": http_status}) + if "error" in parsed_response: + raise HitbtcAPIError(parsed_response) + return parsed_response + + def get_order_price_quantum(self, trading_pair: str, price: Decimal): + """ + Returns a price step, a minimum price increment for a given trading pair. + """ + trading_rule = self._trading_rules[trading_pair] + return trading_rule.min_price_increment + + def get_order_size_quantum(self, trading_pair: str, order_size: Decimal): + """ + Returns an order amount step, a minimum amount increment for a given trading pair. + """ + trading_rule = self._trading_rules[trading_pair] + return Decimal(trading_rule.min_base_amount_increment) + + def get_order_book(self, trading_pair: str) -> OrderBook: + if trading_pair not in self._order_book_tracker.order_books: + raise ValueError(f"No order book exists for '{trading_pair}'.") + return self._order_book_tracker.order_books[trading_pair] + + def buy(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, + price: Decimal = s_decimal_NaN, **kwargs) -> str: + """ + Buys an amount of base asset (of the given trading pair). This function returns immediately. + To see an actual order, you'll have to wait for BuyOrderCreatedEvent. + :param trading_pair: The market (e.g. BTC-USDT) to buy from + :param amount: The amount in base token value + :param order_type: The order type + :param price: The price (note: this is no longer optional) + :returns A new internal order id + """ + order_id: str = get_new_client_order_id(True, trading_pair) + safe_ensure_future(self._create_order(TradeType.BUY, order_id, trading_pair, amount, order_type, price)) + return order_id + + def sell(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, + price: Decimal = s_decimal_NaN, **kwargs) -> str: + """ + Sells an amount of base asset (of the given trading pair). This function returns immediately. + To see an actual order, you'll have to wait for SellOrderCreatedEvent. + :param trading_pair: The market (e.g. BTC-USDT) to sell from + :param amount: The amount in base token value + :param order_type: The order type + :param price: The price (note: this is no longer optional) + :returns A new internal order id + """ + order_id: str = get_new_client_order_id(False, trading_pair) + safe_ensure_future(self._create_order(TradeType.SELL, order_id, trading_pair, amount, order_type, price)) + return order_id + + def cancel(self, trading_pair: str, order_id: str): + """ + Cancel an order. This function returns immediately. + To get the cancellation result, you'll have to wait for OrderCancelledEvent. + :param trading_pair: The market (e.g. BTC-USDT) of the order. + :param order_id: The internal order id (also called client_order_id) + """ + safe_ensure_future(self._execute_cancel(trading_pair, order_id)) + return order_id + + async def _create_order(self, + trade_type: TradeType, + order_id: str, + trading_pair: str, + amount: Decimal, + order_type: OrderType, + price: Decimal): + """ + Calls create-order API end point to place an order, starts tracking the order and triggers order created event. + :param trade_type: BUY or SELL + :param order_id: Internal order id (also called client_order_id) + :param trading_pair: The market to place order + :param amount: The order amount (in base token value) + :param order_type: The order type + :param price: The order price + """ + if not order_type.is_limit_type(): + raise Exception(f"Unsupported order type: {order_type}") + trading_rule = self._trading_rules[trading_pair] + + amount = self.quantize_order_amount(trading_pair, amount) + price = self.quantize_order_price(trading_pair, price) + if amount < trading_rule.min_order_size: + raise ValueError(f"Buy order amount {amount} is lower than the minimum order size " + f"{trading_rule.min_order_size}.") + order_type_str = order_type.name.lower().split("_")[0] + api_params = {"symbol": convert_to_exchange_trading_pair(trading_pair), + "side": trade_type.name.lower(), + "type": order_type_str, + "price": f"{price:f}", + "quantity": f"{amount:f}", + "clientOrderId": order_id, + # Without strict validate, HitBTC might adjust order prices/sizes. + "strictValidate": "true", + } + if order_type is OrderType.LIMIT_MAKER: + api_params["postOnly"] = "true" + self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, order_type) + try: + order_result = await self._api_request("POST", Constants.ENDPOINT["ORDER_CREATE"], api_params, True) + exchange_order_id = str(order_result["id"]) + tracked_order = self._in_flight_orders.get(order_id) + if tracked_order is not None: + self.logger().info(f"Created {order_type.name} {trade_type.name} order {order_id} for " + f"{amount} {trading_pair}.") + tracked_order.update_exchange_order_id(exchange_order_id) + if trade_type is TradeType.BUY: + event_tag = MarketEvent.BuyOrderCreated + event_cls = BuyOrderCreatedEvent + else: + event_tag = MarketEvent.SellOrderCreated + event_cls = SellOrderCreatedEvent + self.trigger_event(event_tag, + event_cls(self.current_timestamp, order_type, trading_pair, amount, price, order_id)) + except asyncio.CancelledError: + raise + except HitbtcAPIError as e: + error_reason = e.error_payload.get('error', {}).get('message') + self.stop_tracking_order(order_id) + self.logger().network( + f"Error submitting {trade_type.name} {order_type.name} order to {Constants.EXCHANGE_NAME} for " + f"{amount} {trading_pair} {price} - {error_reason}.", + exc_info=True, + app_warning_msg=(f"Error submitting order to {Constants.EXCHANGE_NAME} - {error_reason}.") + ) + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent(self.current_timestamp, order_id, order_type)) + + def start_tracking_order(self, + order_id: str, + exchange_order_id: str, + trading_pair: str, + trade_type: TradeType, + price: Decimal, + amount: Decimal, + order_type: OrderType): + """ + Starts tracking an order by simply adding it into _in_flight_orders dictionary. + """ + self._in_flight_orders[order_id] = HitbtcInFlightOrder( + client_order_id=order_id, + exchange_order_id=exchange_order_id, + trading_pair=trading_pair, + order_type=order_type, + trade_type=trade_type, + price=price, + amount=amount + ) + + def stop_tracking_order(self, order_id: str): + """ + Stops tracking an order by simply removing it from _in_flight_orders dictionary. + """ + if order_id in self._in_flight_orders: + del self._in_flight_orders[order_id] + if order_id in self._order_not_found_records: + del self._order_not_found_records[order_id] + + async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: + """ + Executes order cancellation process by first calling cancel-order API. The API result doesn't confirm whether + the cancellation is successful, it simply states it receives the request. + :param trading_pair: The market trading pair (Unused during cancel on HitBTC) + :param order_id: The internal order id + order.last_state to change to CANCELED + """ + order_was_cancelled = False + try: + tracked_order = self._in_flight_orders.get(order_id) + if tracked_order is None: + raise ValueError(f"Failed to cancel order - {order_id}. Order not found.") + if tracked_order.exchange_order_id is None: + await tracked_order.get_exchange_order_id() + # ex_order_id = tracked_order.exchange_order_id + await self._api_request("DELETE", + Constants.ENDPOINT["ORDER_DELETE"].format(id=order_id), + is_auth_required=True) + order_was_cancelled = True + except asyncio.CancelledError: + raise + except HitbtcAPIError as e: + err = e.error_payload.get('error', e.error_payload) + self._order_not_found_records[order_id] = self._order_not_found_records.get(order_id, 0) + 1 + if err.get('code') == 20002 and \ + self._order_not_found_records[order_id] >= self.ORDER_NOT_EXIST_CANCEL_COUNT: + order_was_cancelled = True + if order_was_cancelled: + self.logger().info(f"Successfully cancelled order {order_id} on {Constants.EXCHANGE_NAME}.") + self.stop_tracking_order(order_id) + self.trigger_event(MarketEvent.OrderCancelled, + OrderCancelledEvent(self.current_timestamp, order_id)) + tracked_order.cancelled_event.set() + return CancellationResult(order_id, True) + else: + self.logger().network( + f"Failed to cancel order {order_id}: {err.get('message', str(err))}", + exc_info=True, + app_warning_msg=f"Failed to cancel the order {order_id} on {Constants.EXCHANGE_NAME}. " + f"Check API key and network connection." + ) + return CancellationResult(order_id, False) + + async def _status_polling_loop(self): + """ + Periodically update user balances and order status via REST API. This serves as a fallback measure for web + socket API updates. + """ + while True: + try: + self._poll_notifier = asyncio.Event() + await self._poll_notifier.wait() + await safe_gather( + self._update_balances(), + self._update_order_status(), + ) + self._last_poll_timestamp = self.current_timestamp + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().error(str(e), exc_info=True) + warn_msg = (f"Could not fetch account updates from {Constants.EXCHANGE_NAME}. " + "Check API key and network connection.") + self.logger().network("Unexpected error while fetching account updates.", exc_info=True, + app_warning_msg=warn_msg) + await asyncio.sleep(0.5) + + async def _update_balances(self): + """ + Calls REST API to update total and available balances. + """ + account_info = await self._api_request("GET", Constants.ENDPOINT["USER_BALANCES"], is_auth_required=True) + self._process_balance_message(account_info) + + async def _update_order_status(self): + """ + Calls REST API to get status update for each in-flight order. + """ + last_tick = int(self._last_poll_timestamp / Constants.UPDATE_ORDER_STATUS_INTERVAL) + current_tick = int(self.current_timestamp / Constants.UPDATE_ORDER_STATUS_INTERVAL) + + if current_tick > last_tick and len(self._in_flight_orders) > 0: + tracked_orders = list(self._in_flight_orders.values()) + tasks = [] + for tracked_order in tracked_orders: + # exchange_order_id = await tracked_order.get_exchange_order_id() + order_id = tracked_order.client_order_id + tasks.append(self._api_request("GET", + Constants.ENDPOINT["ORDER_STATUS"].format(id=order_id), + is_auth_required=True)) + self.logger().debug(f"Polling for order status updates of {len(tasks)} orders.") + responses = await safe_gather(*tasks, return_exceptions=True) + for response, tracked_order in zip(responses, tracked_orders): + client_order_id = tracked_order.client_order_id + if isinstance(response, HitbtcAPIError): + err = response.error_payload.get('error', response.error_payload) + if err.get('code') == 20002: + self._order_not_found_records[client_order_id] = \ + self._order_not_found_records.get(client_order_id, 0) + 1 + if self._order_not_found_records[client_order_id] < self.ORDER_NOT_EXIST_CONFIRMATION_COUNT: + # Wait until the order not found error have repeated a few times before actually treating + # it as failed. See: https://github.com/CoinAlpha/hummingbot/issues/601 + continue + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent( + self.current_timestamp, client_order_id, tracked_order.order_type)) + self.stop_tracking_order(client_order_id) + else: + continue + elif "clientOrderId" not in response: + self.logger().info(f"_update_order_status clientOrderId not in resp: {response}") + continue + else: + self._process_order_message(response) + + def _process_order_message(self, order_msg: Dict[str, Any]): + """ + Updates in-flight order and triggers cancellation or failure event if needed. + :param order_msg: The order response from either REST or web socket API (they are of the same format) + Example Order: + { + "id": "4345613661", + "clientOrderId": "57d5525562c945448e3cbd559bd068c3", + "symbol": "BCCBTC", + "side": "sell", + "status": "new", + "type": "limit", + "timeInForce": "GTC", + "quantity": "0.013", + "price": "0.100000", + "cumQuantity": "0.000", + "postOnly": false, + "createdAt": "2017-10-20T12:17:12.245Z", + "updatedAt": "2017-10-20T12:17:12.245Z", + "reportType": "status" + } + """ + client_order_id = order_msg["clientOrderId"] + if client_order_id not in self._in_flight_orders: + return + tracked_order = self._in_flight_orders[client_order_id] + # Update order execution status + tracked_order.last_state = order_msg["status"] + # update order + tracked_order.executed_amount_base = Decimal(order_msg["cumQuantity"]) + tracked_order.executed_amount_quote = Decimal(order_msg["price"]) * Decimal(order_msg["cumQuantity"]) + + if tracked_order.is_cancelled: + self.logger().info(f"Successfully cancelled order {client_order_id}.") + self.stop_tracking_order(client_order_id) + self.trigger_event(MarketEvent.OrderCancelled, + OrderCancelledEvent(self.current_timestamp, client_order_id)) + tracked_order.cancelled_event.set() + elif tracked_order.is_failure: + self.logger().info(f"The market order {client_order_id} has failed according to order status API. ") + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent( + self.current_timestamp, client_order_id, tracked_order.order_type)) + self.stop_tracking_order(client_order_id) + + async def _process_trade_message(self, trade_msg: Dict[str, Any]): + """ + Updates in-flight order and trigger order filled event for trade message received. Triggers order completed + event if the total executed amount equals to the specified order amount. + Example Trade: + { + "id": "4345697765", + "clientOrderId": "53b7cf917963464a811a4af426102c19", + "symbol": "ETHBTC", + "side": "sell", + "status": "filled", + "type": "limit", + "timeInForce": "GTC", + "quantity": "0.001", + "price": "0.053868", + "cumQuantity": "0.001", + "postOnly": false, + "createdAt": "2017-10-20T12:20:05.952Z", + "updatedAt": "2017-10-20T12:20:38.708Z", + "reportType": "trade", + "tradeQuantity": "0.001", + "tradePrice": "0.053868", + "tradeId": 55051694, + "tradeFee": "-0.000000005" + } + """ + tracked_orders = list(self._in_flight_orders.values()) + for order in tracked_orders: + await order.get_exchange_order_id() + track_order = [o for o in tracked_orders if trade_msg["id"] == o.exchange_order_id] + if not track_order: + return + tracked_order = track_order[0] + updated = tracked_order.update_with_trade_update(trade_msg) + if not updated: + return + self.trigger_event( + MarketEvent.OrderFilled, + OrderFilledEvent( + self.current_timestamp, + tracked_order.client_order_id, + tracked_order.trading_pair, + tracked_order.trade_type, + tracked_order.order_type, + Decimal(str(trade_msg.get("tradePrice", "0"))), + Decimal(str(trade_msg.get("tradeQuantity", "0"))), + TradeFee(0.0, [(tracked_order.quote_asset, Decimal(str(trade_msg.get("tradeFee", "0"))))]), + exchange_trade_id=trade_msg["id"] + ) + ) + if math.isclose(tracked_order.executed_amount_base, tracked_order.amount) or \ + tracked_order.executed_amount_base >= tracked_order.amount or \ + tracked_order.is_done: + tracked_order.last_state = "FILLED" + self.logger().info(f"The {tracked_order.trade_type.name} order " + f"{tracked_order.client_order_id} has completed " + f"according to order status API.") + event_tag = MarketEvent.BuyOrderCompleted if tracked_order.trade_type is TradeType.BUY \ + else MarketEvent.SellOrderCompleted + event_class = BuyOrderCompletedEvent if tracked_order.trade_type is TradeType.BUY \ + else SellOrderCompletedEvent + await asyncio.sleep(0.1) + self.trigger_event(event_tag, + event_class(self.current_timestamp, + tracked_order.client_order_id, + tracked_order.base_asset, + tracked_order.quote_asset, + tracked_order.fee_asset, + tracked_order.executed_amount_base, + tracked_order.executed_amount_quote, + tracked_order.fee_paid, + tracked_order.order_type)) + self.stop_tracking_order(tracked_order.client_order_id) + + def _process_balance_message(self, balance_update): + local_asset_names = set(self._account_balances.keys()) + remote_asset_names = set() + for account in balance_update: + asset_name = account["currency"] + self._account_available_balances[asset_name] = Decimal(str(account["available"])) + self._account_balances[asset_name] = Decimal(str(account["reserved"])) + Decimal(str(account["available"])) + remote_asset_names.add(asset_name) + + asset_names_to_remove = local_asset_names.difference(remote_asset_names) + for asset_name in asset_names_to_remove: + del self._account_available_balances[asset_name] + del self._account_balances[asset_name] + + async def cancel_all(self, timeout_seconds: float) -> List[CancellationResult]: + """ + Cancels all in-flight orders and waits for cancellation results. + Used by bot's top level stop and exit commands (cancelling outstanding orders on exit) + :param timeout_seconds: The timeout at which the operation will be canceled. + :returns List of CancellationResult which indicates whether each order is successfully cancelled. + """ + if self._trading_pairs is None: + raise Exception("cancel_all can only be used when trading_pairs are specified.") + open_orders = [o for o in self._in_flight_orders.values() if not o.is_done] + if len(open_orders) == 0: + return [] + tasks = [self._execute_cancel(o.trading_pair, o.client_order_id) for o in open_orders] + cancellation_results = [] + try: + async with timeout(timeout_seconds): + cancellation_results = await safe_gather(*tasks, return_exceptions=False) + except Exception: + self.logger().network( + "Unexpected error cancelling orders.", exc_info=True, + app_warning_msg=(f"Failed to cancel all orders on {Constants.EXCHANGE_NAME}. " + "Check API key and network connection.") + ) + return cancellation_results + + def tick(self, timestamp: float): + """ + Is called automatically by the clock for each clock's tick (1 second by default). + It checks if status polling task is due for execution. + """ + now = time.time() + poll_interval = (Constants.SHORT_POLL_INTERVAL + if now - self._user_stream_tracker.last_recv_time > 60.0 + else Constants.LONG_POLL_INTERVAL) + last_tick = int(self._last_timestamp / poll_interval) + current_tick = int(timestamp / poll_interval) + if current_tick > last_tick: + if not self._poll_notifier.is_set(): + self._poll_notifier.set() + self._last_timestamp = timestamp + + def get_fee(self, + base_currency: str, + quote_currency: str, + order_type: OrderType, + order_side: TradeType, + amount: Decimal, + price: Decimal = s_decimal_NaN) -> TradeFee: + """ + To get trading fee, this function is simplified by using fee override configuration. Most parameters to this + function are ignore except order_type. Use OrderType.LIMIT_MAKER to specify you want trading fee for + maker order. + """ + is_maker = order_type is OrderType.LIMIT_MAKER + return TradeFee(percent=self.estimate_fee_pct(is_maker)) + + async def _iter_user_event_queue(self) -> AsyncIterable[Dict[str, any]]: + while True: + try: + yield await self._user_stream_tracker.user_stream.get() + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unknown error. Retrying after 1 seconds.", exc_info=True, + app_warning_msg=(f"Could not fetch user events from {Constants.EXCHANGE_NAME}. " + "Check API key and network connection.")) + await asyncio.sleep(1.0) + + async def _user_stream_event_listener(self): + """ + Listens to message in _user_stream_tracker.user_stream queue. The messages are put in by + HitbtcAPIUserStreamDataSource. + """ + async for event_message in self._iter_user_event_queue(): + try: + event_methods = [ + Constants.WS_METHODS["USER_ORDERS"], + Constants.WS_METHODS["USER_TRADES"], + ] + method: str = event_message.get("method", None) + params: str = event_message.get("params", None) + account_balances: list = event_message.get("result", None) + + if method not in event_methods and account_balances is None: + self.logger().error(f"Unexpected message in user stream: {event_message}.", exc_info=True) + continue + if method == Constants.WS_METHODS["USER_TRADES"]: + await self._process_trade_message(params) + elif method == Constants.WS_METHODS["USER_ORDERS"]: + for order_msg in params: + self._process_order_message(order_msg) + elif isinstance(account_balances, list) and "currency" in account_balances[0]: + self._process_balance_message(account_balances) + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error in user stream listener loop.", exc_info=True) + await asyncio.sleep(5.0) + + # This is currently unused, but looks like a future addition. + async def get_open_orders(self) -> List[OpenOrder]: + result = await self._api_request("GET", Constants.ENDPOINT["USER_ORDERS"], is_auth_required=True) + ret_val = [] + for order in result: + if Constants.HBOT_BROKER_ID not in order["clientOrderId"]: + continue + if order["type"] != OrderType.LIMIT.name.lower(): + self.logger().info(f"Unsupported order type found: {order['type']}") + continue + ret_val.append( + OpenOrder( + client_order_id=order["clientOrderId"], + trading_pair=convert_from_exchange_trading_pair(order["symbol"]), + price=Decimal(str(order["price"])), + amount=Decimal(str(order["quantity"])), + executed_amount=Decimal(str(order["cumQuantity"])), + status=order["status"], + order_type=OrderType.LIMIT, + is_buy=True if order["side"].lower() == TradeType.BUY.name.lower() else False, + time=str_date_to_ts(order["createdAt"]), + exchange_order_id=order["id"] + ) + ) + return ret_val diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py b/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py new file mode 100644 index 0000000000..54766be2f1 --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py @@ -0,0 +1,118 @@ +from decimal import Decimal +from typing import ( + Any, + Dict, + Optional, +) +import asyncio +from hummingbot.core.event.events import ( + OrderType, + TradeType +) +from hummingbot.connector.in_flight_order_base import InFlightOrderBase + +s_decimal_0 = Decimal(0) + + +class HitbtcInFlightOrder(InFlightOrderBase): + def __init__(self, + client_order_id: str, + exchange_order_id: Optional[str], + trading_pair: str, + order_type: OrderType, + trade_type: TradeType, + price: Decimal, + amount: Decimal, + initial_state: str = "new"): + super().__init__( + client_order_id, + exchange_order_id, + trading_pair, + order_type, + trade_type, + price, + amount, + initial_state, + ) + self.trade_id_set = set() + self.cancelled_event = asyncio.Event() + + @property + def is_done(self) -> bool: + return self.last_state in {"filled", "canceled", "expired"} + + @property + def is_failure(self) -> bool: + return self.last_state in {"suspended"} + + @property + def is_cancelled(self) -> bool: + return self.last_state in {"canceled", "expired"} + + @classmethod + def from_json(cls, data: Dict[str, Any]) -> InFlightOrderBase: + """ + :param data: json data from API + :return: formatted InFlightOrder + """ + retval = HitbtcInFlightOrder( + data["client_order_id"], + data["exchange_order_id"], + data["trading_pair"], + getattr(OrderType, data["order_type"]), + getattr(TradeType, data["trade_type"]), + Decimal(data["price"]), + Decimal(data["amount"]), + data["last_state"] + ) + retval.executed_amount_base = Decimal(data["executed_amount_base"]) + retval.executed_amount_quote = Decimal(data["executed_amount_quote"]) + retval.fee_asset = data["fee_asset"] + retval.fee_paid = Decimal(data["fee_paid"]) + retval.last_state = data["last_state"] + return retval + + def update_with_trade_update(self, trade_update: Dict[str, Any]) -> bool: + """ + Updates the in flight order with trade update (from private/get-order-detail end point) + return: True if the order gets updated otherwise False + Example Trade: + { + "id": "4345697765", + "clientOrderId": "53b7cf917963464a811a4af426102c19", + "symbol": "ETHBTC", + "side": "sell", + "status": "filled", + "type": "limit", + "timeInForce": "GTC", + "quantity": "0.001", + "price": "0.053868", + "cumQuantity": "0.001", + "postOnly": false, + "createdAt": "2017-10-20T12:20:05.952Z", + "updatedAt": "2017-10-20T12:20:38.708Z", + "reportType": "trade", + } + ... Trade variables are only included after fills. + { + "tradeQuantity": "0.001", + "tradePrice": "0.053868", + "tradeId": 55051694, + "tradeFee": "-0.000000005" + } + """ + self.executed_amount_base = Decimal(str(trade_update["cumQuantity"])) + if self.executed_amount_base <= s_decimal_0: + # No trades executed yet. + return False + trade_id = trade_update["updatedAt"] + if trade_id in self.trade_id_set: + # trade already recorded + return False + self.trade_id_set.add(trade_id) + self.fee_paid += Decimal(str(trade_update.get("tradeFee", "0"))) + self.executed_amount_quote += (Decimal(str(trade_update.get("tradePrice", "0"))) * + Decimal(str(trade_update.get("tradeQuantity", "0")))) + if not self.fee_asset: + self.fee_asset = self.quote_asset + return True diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py new file mode 100644 index 0000000000..1a3c91a121 --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python + +import logging +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants + +from sqlalchemy.engine import RowProxy +from typing import ( + Optional, + Dict, + List, Any) +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_message import ( + OrderBookMessage, OrderBookMessageType +) +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_message import HitbtcOrderBookMessage + +_logger = None + + +class HitbtcOrderBook(OrderBook): + @classmethod + def logger(cls) -> HummingbotLogger: + global _logger + if _logger is None: + _logger = logging.getLogger(__name__) + return _logger + + @classmethod + def snapshot_message_from_exchange(cls, + msg: Dict[str, any], + timestamp: float, + metadata: Optional[Dict] = None): + """ + Convert json snapshot data into standard OrderBookMessage format + :param msg: json snapshot data from live web socket stream + :param timestamp: timestamp attached to incoming data + :return: HitbtcOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + return HitbtcOrderBookMessage( + message_type=OrderBookMessageType.SNAPSHOT, + content=msg, + timestamp=timestamp + ) + + @classmethod + def snapshot_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of snapshot data into standard OrderBookMessage format + :param record: a row of snapshot data from the database + :return: HitbtcOrderBookMessage + """ + return HitbtcOrderBookMessage( + message_type=OrderBookMessageType.SNAPSHOT, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def diff_message_from_exchange(cls, + msg: Dict[str, any], + timestamp: Optional[float] = None, + metadata: Optional[Dict] = None): + """ + Convert json diff data into standard OrderBookMessage format + :param msg: json diff data from live web socket stream + :param timestamp: timestamp attached to incoming data + :return: HitbtcOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + return HitbtcOrderBookMessage( + message_type=OrderBookMessageType.DIFF, + content=msg, + timestamp=timestamp + ) + + @classmethod + def diff_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of diff data into standard OrderBookMessage format + :param record: a row of diff data from the database + :return: HitbtcOrderBookMessage + """ + return HitbtcOrderBookMessage( + message_type=OrderBookMessageType.DIFF, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def trade_message_from_exchange(cls, + msg: Dict[str, Any], + timestamp: Optional[float] = None, + metadata: Optional[Dict] = None): + """ + Convert a trade data into standard OrderBookMessage format + :param record: a trade data from the database + :return: HitbtcOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + msg.update({ + "exchange_order_id": msg.get("id"), + "trade_type": msg.get("side"), + "price": msg.get("price"), + "amount": msg.get("quantity"), + }) + + return HitbtcOrderBookMessage( + message_type=OrderBookMessageType.TRADE, + content=msg, + timestamp=timestamp + ) + + @classmethod + def trade_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of trade data into standard OrderBookMessage format + :param record: a row of trade data from the database + :return: HitbtcOrderBookMessage + """ + return HitbtcOrderBookMessage( + message_type=OrderBookMessageType.TRADE, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def from_snapshot(cls, snapshot: OrderBookMessage): + raise NotImplementedError(Constants.EXCHANGE_NAME + " order book needs to retain individual order data.") + + @classmethod + def restore_from_snapshot_and_diffs(self, snapshot: OrderBookMessage, diffs: List[OrderBookMessage]): + raise NotImplementedError(Constants.EXCHANGE_NAME + " order book needs to retain individual order data.") diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py new file mode 100644 index 0000000000..fdc207d64d --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python + +from typing import ( + Dict, + List, + Optional, +) + +from hummingbot.core.data_type.order_book_row import OrderBookRow +from hummingbot.core.data_type.order_book_message import ( + OrderBookMessage, + OrderBookMessageType, +) +from .hitbtc_constants import Constants +from .hitbtc_utils import ( + convert_from_exchange_trading_pair, +) + + +class HitbtcOrderBookMessage(OrderBookMessage): + def __new__( + cls, + message_type: OrderBookMessageType, + content: Dict[str, any], + timestamp: Optional[float] = None, + *args, + **kwargs, + ): + if timestamp is None: + if message_type is OrderBookMessageType.SNAPSHOT: + raise ValueError("timestamp must not be None when initializing snapshot messages.") + timestamp = content["timestamp"] + + return super(HitbtcOrderBookMessage, cls).__new__( + cls, message_type, content, timestamp=timestamp, *args, **kwargs + ) + + @property + def update_id(self) -> int: + if self.type in [OrderBookMessageType.DIFF, OrderBookMessageType.SNAPSHOT]: + return int(self.timestamp * 1e3) + else: + return -1 + + @property + def trade_id(self) -> int: + if self.type is OrderBookMessageType.TRADE: + return int(self.timestamp * 1e3) + return -1 + + @property + def trading_pair(self) -> str: + if "trading_pair" in self.content: + return self.content["trading_pair"] + elif "symbol" in self.content: + return convert_from_exchange_trading_pair(self.content["symbol"]) + + # The `asks` and `bids` properties are only used in the methods below. + # They are all replaced or unused in this connector: + # OrderBook.restore_from_snapshot_and_diffs + # OrderBookTracker._track_single_book + # MockAPIOrderBookDataSource.get_tracking_pairs + @property + def asks(self) -> List[OrderBookRow]: + raise NotImplementedError(Constants.EXCHANGE_NAME + " order book uses active_order_tracker.") + + @property + def bids(self) -> List[OrderBookRow]: + raise NotImplementedError(Constants.EXCHANGE_NAME + " order book uses active_order_tracker.") + + def __eq__(self, other) -> bool: + return self.type == other.type and self.timestamp == other.timestamp + + def __lt__(self, other) -> bool: + if self.timestamp != other.timestamp: + return self.timestamp < other.timestamp + else: + """ + If timestamp is the same, the ordering is snapshot < diff < trade + """ + return self.type.value < other.type.value diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py new file mode 100644 index 0000000000..d3161de17e --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +import asyncio +import bisect +import logging +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants +import time + +from collections import defaultdict, deque +from typing import Optional, Dict, List, Deque +from hummingbot.core.data_type.order_book_message import OrderBookMessageType +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.order_book_tracker import OrderBookTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_message import HitbtcOrderBookMessage +from hummingbot.connector.exchange.hitbtc.hitbtc_active_order_tracker import HitbtcActiveOrderTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_api_order_book_data_source import HitbtcAPIOrderBookDataSource +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book import HitbtcOrderBook + + +class HitbtcOrderBookTracker(OrderBookTracker): + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, trading_pairs: Optional[List[str]] = None,): + super().__init__(HitbtcAPIOrderBookDataSource(trading_pairs), trading_pairs) + + self._ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + self._order_book_snapshot_stream: asyncio.Queue = asyncio.Queue() + self._order_book_diff_stream: asyncio.Queue = asyncio.Queue() + self._order_book_trade_stream: asyncio.Queue = asyncio.Queue() + self._process_msg_deque_task: Optional[asyncio.Task] = None + self._past_diffs_windows: Dict[str, Deque] = {} + self._order_books: Dict[str, HitbtcOrderBook] = {} + self._saved_message_queues: Dict[str, Deque[HitbtcOrderBookMessage]] = \ + defaultdict(lambda: deque(maxlen=1000)) + self._active_order_trackers: Dict[str, HitbtcActiveOrderTracker] = defaultdict(HitbtcActiveOrderTracker) + self._order_book_stream_listener_task: Optional[asyncio.Task] = None + self._order_book_trade_listener_task: Optional[asyncio.Task] = None + + @property + def exchange_name(self) -> str: + """ + Name of the current exchange + """ + return Constants.EXCHANGE_NAME + + async def _track_single_book(self, trading_pair: str): + """ + Update an order book with changes from the latest batch of received messages + """ + past_diffs_window: Deque[HitbtcOrderBookMessage] = deque() + self._past_diffs_windows[trading_pair] = past_diffs_window + + message_queue: asyncio.Queue = self._tracking_message_queues[trading_pair] + order_book: HitbtcOrderBook = self._order_books[trading_pair] + active_order_tracker: HitbtcActiveOrderTracker = self._active_order_trackers[trading_pair] + + last_message_timestamp: float = time.time() + diff_messages_accepted: int = 0 + + while True: + try: + message: HitbtcOrderBookMessage = None + saved_messages: Deque[HitbtcOrderBookMessage] = self._saved_message_queues[trading_pair] + # Process saved messages first if there are any + if len(saved_messages) > 0: + message = saved_messages.popleft() + else: + message = await message_queue.get() + + if message.type is OrderBookMessageType.DIFF: + bids, asks = active_order_tracker.convert_diff_message_to_order_book_row(message) + order_book.apply_diffs(bids, asks, message.update_id) + past_diffs_window.append(message) + while len(past_diffs_window) > self.PAST_DIFF_WINDOW_SIZE: + past_diffs_window.popleft() + diff_messages_accepted += 1 + + # Output some statistics periodically. + now: float = time.time() + if int(now / 60.0) > int(last_message_timestamp / 60.0): + self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") + diff_messages_accepted = 0 + last_message_timestamp = now + elif message.type is OrderBookMessageType.SNAPSHOT: + past_diffs: List[HitbtcOrderBookMessage] = list(past_diffs_window) + # only replay diffs later than snapshot, first update active order with snapshot then replay diffs + replay_position = bisect.bisect_right(past_diffs, message) + replay_diffs = past_diffs[replay_position:] + s_bids, s_asks = active_order_tracker.convert_snapshot_message_to_order_book_row(message) + order_book.apply_snapshot(s_bids, s_asks, message.update_id) + for diff_message in replay_diffs: + d_bids, d_asks = active_order_tracker.convert_diff_message_to_order_book_row(diff_message) + order_book.apply_diffs(d_bids, d_asks, diff_message.update_id) + + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + f"Unexpected error processing order book messages for {trading_pair}.", + exc_info=True, + app_warning_msg="Unexpected error processing order book messages. Retrying after 5 seconds." + ) + await asyncio.sleep(5.0) diff --git a/hummingbot/connector/exchange/bitmax/bitmax_order_book_tracker_entry.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker_entry.py similarity index 56% rename from hummingbot/connector/exchange/bitmax/bitmax_order_book_tracker_entry.py rename to hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker_entry.py index a97a33088a..5edfbadec0 100644 --- a/hummingbot/connector/exchange/bitmax/bitmax_order_book_tracker_entry.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker_entry.py @@ -1,21 +1,21 @@ from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.order_book_tracker_entry import OrderBookTrackerEntry -from hummingbot.connector.exchange.bitmax.bitmax_active_order_tracker import BitmaxActiveOrderTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_active_order_tracker import HitbtcActiveOrderTracker -class BitmaxOrderBookTrackerEntry(OrderBookTrackerEntry): +class HitbtcOrderBookTrackerEntry(OrderBookTrackerEntry): def __init__( - self, trading_pair: str, timestamp: float, order_book: OrderBook, active_order_tracker: BitmaxActiveOrderTracker + self, trading_pair: str, timestamp: float, order_book: OrderBook, active_order_tracker: HitbtcActiveOrderTracker ): self._active_order_tracker = active_order_tracker - super(BitmaxOrderBookTrackerEntry, self).__init__(trading_pair, timestamp, order_book) + super(HitbtcOrderBookTrackerEntry, self).__init__(trading_pair, timestamp, order_book) def __repr__(self) -> str: return ( - f"BitmaxOrderBookTrackerEntry(trading_pair='{self._trading_pair}', timestamp='{self._timestamp}', " + f"HitbtcOrderBookTrackerEntry(trading_pair='{self._trading_pair}', timestamp='{self._timestamp}', " f"order_book='{self._order_book}')" ) @property - def active_order_tracker(self) -> BitmaxActiveOrderTracker: + def active_order_tracker(self) -> HitbtcActiveOrderTracker: return self._active_order_tracker diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_user_stream_tracker.py b/hummingbot/connector/exchange/hitbtc/hitbtc_user_stream_tracker.py new file mode 100644 index 0000000000..7b04002ccd --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_user_stream_tracker.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +import asyncio +import logging +from typing import ( + Optional, + List, +) +from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.user_stream_tracker import ( + UserStreamTracker +) +from hummingbot.core.utils.async_utils import ( + safe_ensure_future, + safe_gather, +) +from hummingbot.connector.exchange.hitbtc.hitbtc_api_user_stream_data_source import \ + HitbtcAPIUserStreamDataSource +from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants + + +class HitbtcUserStreamTracker(UserStreamTracker): + _cbpust_logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._bust_logger is None: + cls._bust_logger = logging.getLogger(__name__) + return cls._bust_logger + + def __init__(self, + hitbtc_auth: Optional[HitbtcAuth] = None, + trading_pairs: Optional[List[str]] = []): + super().__init__() + self._hitbtc_auth: HitbtcAuth = hitbtc_auth + self._trading_pairs: List[str] = trading_pairs + self._ev_loop: asyncio.events.AbstractEventLoop = asyncio.get_event_loop() + self._data_source: Optional[UserStreamTrackerDataSource] = None + self._user_stream_tracking_task: Optional[asyncio.Task] = None + + @property + def data_source(self) -> UserStreamTrackerDataSource: + """ + *required + Initializes a user stream data source (user specific order diffs from live socket stream) + :return: OrderBookTrackerDataSource + """ + if not self._data_source: + self._data_source = HitbtcAPIUserStreamDataSource( + hitbtc_auth=self._hitbtc_auth, + trading_pairs=self._trading_pairs + ) + return self._data_source + + @property + def exchange_name(self) -> str: + """ + *required + Name of the current exchange + """ + return Constants.EXCHANGE_NAME + + async def start(self): + """ + *required + Start all listeners and tasks + """ + self._user_stream_tracking_task = safe_ensure_future( + self.data_source.listen_for_user_stream(self._ev_loop, self._user_stream) + ) + await safe_gather(self._user_stream_tracking_task) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py new file mode 100644 index 0000000000..66a3de90cf --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py @@ -0,0 +1,170 @@ +import aiohttp +import asyncio +import random +import re +from dateutil.parser import parse as dateparse +from typing import ( + Any, + Dict, + Optional, + Tuple, +) + +from hummingbot.core.utils.tracking_nonce import get_tracking_nonce +from hummingbot.client.config.config_var import ConfigVar +from hummingbot.client.config.config_methods import using_exchange +from .hitbtc_constants import Constants + + +TRADING_PAIR_SPLITTER = re.compile(Constants.TRADING_PAIR_SPLITTER) + +CENTRALIZED = True + +EXAMPLE_PAIR = "BTC-USD" + +DEFAULT_FEES = [0.1, 0.25] + + +class HitbtcAPIError(IOError): + def __init__(self, error_payload: Dict[str, Any]): + super().__init__(str(error_payload)) + self.error_payload = error_payload + + +# convert date string to timestamp +def str_date_to_ts(date: str) -> int: + return int(dateparse(date).timestamp()) + + +# Request ID class +class RequestId: + """ + Generate request ids + """ + _request_id: int = 0 + + @classmethod + def generate_request_id(cls) -> int: + return get_tracking_nonce() + + +def split_trading_pair(trading_pair: str) -> Optional[Tuple[str, str]]: + try: + m = TRADING_PAIR_SPLITTER.match(trading_pair) + return m.group(1), m.group(2) + # Exceptions are now logged as warnings in trading pair fetcher + except Exception: + return None + + +def translate_tokens(hb_trading_pair: str) -> str: + token_replacements = [ + ("USD", "USDT"), + ] + tokens = hb_trading_pair.split('-') + for token_replacement in token_replacements: + for x in range(len(tokens)): + for inv in [0, 1]: + if tokens[x] == token_replacement[inv]: + tokens[x] = token_replacement[(0 if inv else 1)] + break + return '-'.join(tokens) + + +def convert_from_exchange_trading_pair(ex_trading_pair: str) -> Optional[str]: + regex_match = split_trading_pair(ex_trading_pair) + if regex_match is None: + return None + # HitBTC uses uppercase (BTCUSDT) + base_asset, quote_asset = split_trading_pair(ex_trading_pair) + return translate_tokens(f"{base_asset.upper()}-{quote_asset.upper()}") + + +def convert_to_exchange_trading_pair(hb_trading_pair: str) -> str: + # HitBTC uses uppercase (BTCUSDT) + return translate_tokens(hb_trading_pair).replace("-", "").upper() + + +def get_new_client_order_id(is_buy: bool, trading_pair: str) -> str: + side = "B" if is_buy else "S" + symbols = trading_pair.split("-") + base = symbols[0].upper() + quote = symbols[1].upper() + base_str = f"{base[0]}{base[-1]}" + quote_str = f"{quote[0]}{quote[-1]}" + return f"{Constants.HBOT_BROKER_ID}-{side}-{base_str}{quote_str}-{get_tracking_nonce()}" + + +def retry_sleep_time(try_count: int) -> float: + random.seed() + randSleep = 1 + float(random.randint(1, 10) / 100) + return float(2 + float(randSleep * (1 + (try_count ** try_count)))) + + +async def aiohttp_response_with_errors(request_coroutine): + http_status, parsed_response, request_errors = None, None, False + try: + async with request_coroutine as response: + http_status = response.status + try: + parsed_response = await response.json() + except Exception: + request_errors = True + try: + parsed_response = str(await response.read()) + if len(parsed_response) > 100: + parsed_response = f"{parsed_response[:100]} ... (truncated)" + except Exception: + pass + TempFailure = (parsed_response is None or + (response.status not in [200, 201] and "error" not in parsed_response)) + if TempFailure: + parsed_response = response.reason if parsed_response is None else parsed_response + request_errors = True + except Exception: + request_errors = True + return http_status, parsed_response, request_errors + + +async def api_call_with_retries(method, + endpoint, + params: Optional[Dict[str, Any]] = None, + shared_client=None, + try_count: int = 0) -> Dict[str, Any]: + url = f"{Constants.REST_URL}/{endpoint}" + headers = {"Content-Type": "application/json"} + http_client = shared_client if shared_client is not None else aiohttp.ClientSession() + # Build request coro + response_coro = http_client.request(method=method.upper(), url=url, headers=headers, + params=params, timeout=Constants.API_CALL_TIMEOUT) + http_status, parsed_response, request_errors = await aiohttp_response_with_errors(response_coro) + if shared_client is None: + await http_client.close() + if request_errors or parsed_response is None: + if try_count < Constants.API_MAX_RETRIES: + try_count += 1 + time_sleep = retry_sleep_time(try_count) + print(f"Error fetching data from {url}. HTTP status is {http_status}. " + f"Retrying in {time_sleep:.0f}s.") + await asyncio.sleep(time_sleep) + return await api_call_with_retries(method=method, endpoint=endpoint, params=params, + shared_client=shared_client, try_count=try_count) + else: + raise HitbtcAPIError({"error": parsed_response, "status": http_status}) + return parsed_response + + +KEYS = { + "hitbtc_api_key": + ConfigVar(key="hitbtc_api_key", + prompt=f"Enter your {Constants.EXCHANGE_NAME} API key >>> ", + required_if=using_exchange("hitbtc"), + is_secure=True, + is_connect_key=True), + "hitbtc_secret_key": + ConfigVar(key="hitbtc_secret_key", + prompt=f"Enter your {Constants.EXCHANGE_NAME} secret key >>> ", + required_if=using_exchange("hitbtc"), + is_secure=True, + is_connect_key=True), +} diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py new file mode 100644 index 0000000000..da65b869a2 --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +import asyncio +import copy +import logging +import websockets +import json +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants + + +from typing import ( + Any, + AsyncIterable, + Dict, + Optional, +) +from websockets.exceptions import ConnectionClosed +from hummingbot.logger import HummingbotLogger +from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth +from hummingbot.connector.exchange.hitbtc.hitbtc_utils import ( + RequestId, + HitbtcAPIError, +) + +# reusable websocket class +# ToDo: We should eventually remove this class, and instantiate web socket connection normally (see Binance for example) + + +class HitbtcWebsocket(RequestId): + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, + auth: Optional[HitbtcAuth] = None): + self._auth: Optional[HitbtcAuth] = auth + self._isPrivate = True if self._auth is not None else False + self._WS_URL = Constants.WS_PRIVATE_URL if self._isPrivate else Constants.WS_PUBLIC_URL + self._client: Optional[websockets.WebSocketClientProtocol] = None + + # connect to exchange + async def connect(self): + self._client = await websockets.connect(self._WS_URL) + + # if auth class was passed into websocket class + # we need to emit authenticated requests + if self._isPrivate: + auth_params = self._auth.generate_auth_dict_ws(self.generate_request_id()) + await self._emit("login", auth_params, no_id=True) + raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=Constants.MESSAGE_TIMEOUT) + json_msg = json.loads(raw_msg_str) + if json_msg.get("result") is not True: + err_msg = json_msg.get('error', {}).get('message') + raise HitbtcAPIError({"error": f"Failed to authenticate to websocket - {err_msg}."}) + + return self._client + + # disconnect from exchange + async def disconnect(self): + if self._client is None: + return + + await self._client.close() + + # receive & parse messages + async def _messages(self) -> AsyncIterable[Any]: + try: + while True: + try: + raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=Constants.MESSAGE_TIMEOUT) + try: + msg = json.loads(raw_msg_str) + # HitBTC doesn't support ping or heartbeat messages. + # Can handle them here if that changes - use `safe_ensure_future`. + yield msg + except ValueError: + continue + except asyncio.TimeoutError: + await asyncio.wait_for(self._client.ping(), timeout=Constants.PING_TIMEOUT) + except asyncio.TimeoutError: + self.logger().warning("WebSocket ping timed out. Going to reconnect...") + return + except ConnectionClosed: + return + finally: + await self.disconnect() + + # emit messages + async def _emit(self, method: str, data: Optional[Dict[str, Any]] = {}, no_id: bool = False) -> int: + id = self.generate_request_id() + + payload = { + "id": id, + "method": method, + "params": copy.deepcopy(data), + } + + await self._client.send(json.dumps(payload)) + + return id + + # request via websocket + async def request(self, method: str, data: Optional[Dict[str, Any]] = {}) -> int: + return await self._emit(method, data) + + # subscribe to a method + async def subscribe(self, + channel: str, + trading_pair: Optional[str] = None, + params: Optional[Dict[str, Any]] = {}) -> int: + if trading_pair is not None: + params['symbol'] = trading_pair + return await self.request(f"subscribe{channel}", params) + + # unsubscribe to a method + async def unsubscribe(self, + channel: str, + trading_pair: Optional[str] = None, + params: Optional[Dict[str, Any]] = {}) -> int: + if trading_pair is not None: + params['symbol'] = trading_pair + return await self.request(f"unsubscribe{channel}", params) + + # listen to messages by method + async def on_message(self) -> AsyncIterable[Any]: + async for msg in self._messages(): + yield msg diff --git a/hummingbot/connector/exchange/kraken/kraken_api_order_book_data_source.py b/hummingbot/connector/exchange/kraken/kraken_api_order_book_data_source.py index bc5d2c28e9..b3ee16a409 100755 --- a/hummingbot/connector/exchange/kraken/kraken_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/kraken/kraken_api_order_book_data_source.py @@ -11,15 +11,11 @@ List, Optional ) -from decimal import Decimal import time import ujson import websockets from websockets.exceptions import ConnectionClosed -import requests -import cachetools.func - from hummingbot.core.utils.async_utils import safe_gather from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource from hummingbot.core.data_type.order_book_message import OrderBookMessage @@ -130,21 +126,6 @@ async def _inner_messages(self, finally: await ws.close() - @staticmethod - @cachetools.func.ttl_cache(ttl=10) - def get_mid_price(trading_pair: str) -> Optional[Decimal]: - from hummingbot.connector.exchange.kraken.kraken_utils import convert_to_exchange_trading_pair - - KRAKEN_PRICE_URL = "https://api.kraken.com/0/public/Ticker?pair=" - k_pair = convert_to_exchange_trading_pair(trading_pair) - resp = requests.get(url=KRAKEN_PRICE_URL + k_pair) - resp_json = resp.json() - if len(resp_json["error"]) == 0: - for record in resp_json["result"]: # assume only one pair is received - record = resp_json["result"][record] - result = (Decimal(record["a"][0]) + Decimal(record["b"][0])) / Decimal("2") - return result - @staticmethod async def fetch_trading_pairs() -> List[str]: try: diff --git a/hummingbot/connector/exchange/kucoin/kucoin_api_order_book_data_source.py b/hummingbot/connector/exchange/kucoin/kucoin_api_order_book_data_source.py index cb49ea881f..f46aa80358 100644 --- a/hummingbot/connector/exchange/kucoin/kucoin_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/kucoin/kucoin_api_order_book_data_source.py @@ -4,14 +4,11 @@ import aiohttp import asyncio from async_timeout import timeout -import cachetools.func from collections import defaultdict -from decimal import Decimal from enum import Enum import json import logging import pandas as pd -import requests import time from typing import ( Any, @@ -319,18 +316,6 @@ async def get_trading_pairs(self) -> List[str]: ) return self._trading_pairs - @staticmethod - @cachetools.func.ttl_cache(ttl=10) - def get_mid_price(trading_pair: str) -> Optional[Decimal]: - resp = requests.get(url=TICKER_PRICE_CHANGE_URL) - records = resp.json() - result = None - for record in records["data"]["ticker"]: - if trading_pair == record["symbolName"] and record["buy"] is not None and record["sell"] is not None: - result = (Decimal(record["buy"]) + Decimal(record["sell"])) / Decimal("2") - break - return result - @staticmethod async def fetch_trading_pairs() -> List[str]: async with aiohttp.ClientSession() as client: diff --git a/hummingbot/connector/exchange/liquid/liquid_api_order_book_data_source.py b/hummingbot/connector/exchange/liquid/liquid_api_order_book_data_source.py index 454686452c..3e01510d95 100644 --- a/hummingbot/connector/exchange/liquid/liquid_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/liquid/liquid_api_order_book_data_source.py @@ -4,10 +4,7 @@ import pandas as pd import time from typing import Any, AsyncIterable, Dict, List, Optional -from decimal import Decimal import ujson -import requests -import cachetools.func import websockets from websockets.exceptions import ConnectionClosed @@ -162,19 +159,6 @@ def filter_market_data(cls, exchange_markets_data) -> (List[dict]): if item['disabled'] is False ] - @staticmethod - @cachetools.func.ttl_cache(ttl=10) - def get_mid_price(trading_pair: str) -> Optional[Decimal]: - resp = requests.get(url=Constants.GET_EXCHANGE_MARKETS_URL) - records = resp.json() - result = None - for record in records: - pair = f"{record['base_currency']}-{record['quoted_currency']}" - if trading_pair == pair and record["market_ask"] is not None and record["market_bid"] is not None: - result = (Decimal(record["market_ask"]) + Decimal(record["market_bid"])) / Decimal("2") - break - return result - @staticmethod async def fetch_trading_pairs() -> List[str]: try: diff --git a/hummingbot/connector/exchange/loopring/loopring_api_order_book_data_source.py b/hummingbot/connector/exchange/loopring/loopring_api_order_book_data_source.py index c227739459..8773fb2bbf 100644 --- a/hummingbot/connector/exchange/loopring/loopring_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/loopring/loopring_api_order_book_data_source.py @@ -1,18 +1,9 @@ #!/usr/bin/env python import asyncio -from decimal import Decimal - import aiohttp import logging -# import pandas as pd -# import math - -import requests -import cachetools.func - from typing import AsyncIterable, Dict, List, Optional, Any - import time import ujson import websockets @@ -123,17 +114,6 @@ async def _inner_messages(self, ws: websockets.WebSocketClientProtocol) -> Async finally: await ws.close() - @staticmethod - @cachetools.func.ttl_cache(ttl=10) - def get_mid_price(trading_pair: str) -> Optional[Decimal]: - resp = requests.get(url=LOOPRING_PRICE_URL, params={"market": trading_pair}) - record = resp.json() - if record["resultInfo"]["code"] == 0: - data = record["data"] - mid_price = (Decimal(data[9]) + Decimal(data[10])) / 2 - - return mid_price - @staticmethod async def fetch_trading_pairs() -> List[str]: try: diff --git a/hummingbot/connector/exchange/okex/okex_api_order_book_data_source.py b/hummingbot/connector/exchange/okex/okex_api_order_book_data_source.py index 105982bb79..4aab780846 100644 --- a/hummingbot/connector/exchange/okex/okex_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/okex/okex_api_order_book_data_source.py @@ -14,9 +14,6 @@ List, Optional, ) -from decimal import Decimal -import requests -import cachetools.func import websockets from websockets.exceptions import ConnectionClosed @@ -28,7 +25,6 @@ from hummingbot.connector.exchange.okex.okex_order_book import OkexOrderBook from hummingbot.connector.exchange.okex.constants import ( OKEX_SYMBOLS_URL, - OKEX_PRICE_URL, OKEX_DEPTH_URL, OKEX_WS_URI, ) @@ -80,17 +76,8 @@ async def get_active_exchange_markets(cls) -> pd.DataFrame: all_markets.rename({"quote_volume_24h": "volume", "last": "price"}, axis="columns", inplace=True) - return all_markets - @staticmethod - @cachetools.func.ttl_cache(ttl=10) - def get_mid_price(trading_pair: str) -> Optional[Decimal]: - resp = requests.get(url=OKEX_PRICE_URL.format(trading_pair=trading_pair)) - record = resp.json() - if 'best_ask' in record and 'best_bid' in record: - return (Decimal(record["best_ask"]) + Decimal(record["best_bid"])) / Decimal("2") - @staticmethod async def fetch_trading_pairs() -> List[str]: # Returns a List of str, representing each active trading pair on the exchange. diff --git a/hummingbot/core/rate_oracle/__init__.py b/hummingbot/core/rate_oracle/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hummingbot/core/rate_oracle/rate_oracle.py b/hummingbot/core/rate_oracle/rate_oracle.py new file mode 100644 index 0000000000..14b7d31905 --- /dev/null +++ b/hummingbot/core/rate_oracle/rate_oracle.py @@ -0,0 +1,205 @@ +import asyncio +import logging +from typing import ( + Dict, + Optional, + List +) +from decimal import Decimal +import aiohttp +from enum import Enum +from hummingbot.logger import HummingbotLogger +from hummingbot.core.network_base import NetworkBase, NetworkStatus +from hummingbot.core.utils.async_utils import safe_ensure_future +from hummingbot.connector.exchange.binance.binance_utils import convert_from_exchange_trading_pair as \ + binance_convert_from_exchange_pair +from hummingbot.core.rate_oracle.utils import find_rate +from hummingbot.core.utils.async_utils import safe_gather +from hummingbot.core.utils import async_ttl_cache + + +class RateOracleSource(Enum): + binance = 0 + coingecko = 1 + + +class RateOracle(NetworkBase): + source: RateOracleSource = RateOracleSource.binance + global_token: str = "USDT" + global_token_symbol: str = "$" + _logger: Optional[HummingbotLogger] = None + _shared_instance: "RateOracle" = None + _shared_client: Optional[aiohttp.ClientSession] = None + _cgecko_supported_vs_tokens: List[str] = [] + + binance_price_url = "https://api.binance.com/api/v3/ticker/bookTicker" + coingecko_usd_price_url = "https://api.coingecko.com/api/v3/coins/markets?vs_currency={}&order=market_cap_desc" \ + "&per_page=250&page={}&sparkline=false" + coingecko_supported_vs_tokens_url = "https://api.coingecko.com/api/v3/simple/supported_vs_currencies" + + @classmethod + def get_instance(cls) -> "RateOracle": + if cls._shared_instance is None: + cls._shared_instance = RateOracle() + return cls._shared_instance + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self): + super().__init__() + self._check_network_interval = 30.0 + self._ev_loop = asyncio.get_event_loop() + self._prices: Dict[str, Decimal] = {} + self._fetch_price_task: Optional[asyncio.Task] = None + self._ready_event = asyncio.Event() + + @classmethod + async def _http_client(cls) -> aiohttp.ClientSession: + if cls._shared_client is None: + cls._shared_client = aiohttp.ClientSession() + return cls._shared_client + + async def get_ready(self): + try: + if not self._ready_event.is_set(): + await self._ready_event.wait() + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error while waiting for data feed to get ready.", + exc_info=True) + + @property + def name(self) -> str: + return "rate_oracle" + + @property + def prices(self) -> Dict[str, Decimal]: + return self._prices.copy() + + def update_interval(self) -> float: + return 1.0 + + def rate(self, pair: str) -> Decimal: + return find_rate(self._prices, pair) + + @classmethod + async def rate_async(cls, pair: str) -> Decimal: + prices = await cls.get_prices() + return find_rate(prices, pair) + + @classmethod + async def global_rate(cls, token: str) -> Decimal: + prices = await cls.get_prices() + pair = token + "-" + cls.global_token + return find_rate(prices, pair) + + @classmethod + async def global_value(cls, token: str, amount: Decimal) -> Decimal: + rate = await cls.global_rate(token) + rate = Decimal("0") if rate is None else rate + return amount * rate + + async def fetch_price_loop(self): + while True: + try: + self._prices = await self.get_prices() + if self._prices: + self._ready_event.set() + except asyncio.CancelledError: + raise + except Exception: + self.logger().network(f"Error fetching new prices from {self.source.name}.", exc_info=True, + app_warning_msg=f"Couldn't fetch newest prices from {self.source.name}.") + await asyncio.sleep(self.update_interval()) + + @classmethod + async def get_prices(cls) -> Dict[str, Decimal]: + if cls.source == RateOracleSource.binance: + return await cls.get_binance_prices() + elif cls.source == RateOracleSource.coingecko: + return await cls.get_coingecko_prices(cls.global_token) + else: + raise NotImplementedError + + @classmethod + @async_ttl_cache(ttl=1, maxsize=1) + async def get_binance_prices(cls) -> Dict[str, Decimal]: + results = {} + client = await cls._http_client() + try: + async with client.request("GET", cls.binance_price_url) as resp: + records = await resp.json() + for record in records: + trading_pair = binance_convert_from_exchange_pair(record["symbol"]) + if trading_pair and record["bidPrice"] is not None and record["askPrice"] is not None: + results[trading_pair] = (Decimal(record["bidPrice"]) + Decimal(record["askPrice"])) / Decimal("2") + except asyncio.CancelledError: + raise + except Exception: + cls.logger().error("Unexpected error while retrieving rates from Binance.") + return results + + @classmethod + @async_ttl_cache(ttl=30, maxsize=1) + async def get_coingecko_prices(cls, vs_currency: str) -> Dict[str, Decimal]: + results = {} + if not cls._cgecko_supported_vs_tokens: + client = await cls._http_client() + async with client.request("GET", cls.coingecko_supported_vs_tokens_url) as resp: + records = await resp.json() + cls._cgecko_supported_vs_tokens = records + if vs_currency.lower() not in cls._cgecko_supported_vs_tokens: + vs_currency = "usd" + tasks = [cls.get_coingecko_prices_by_page(vs_currency, i) for i in range(1, 5)] + task_results = await safe_gather(*tasks, return_exceptions=True) + for task_result in task_results: + if isinstance(task_result, Exception): + cls.logger().error("Unexpected error while retrieving rates from Coingecko. " + "Check the log file for more info.") + break + else: + results.update(task_result) + return results + + @classmethod + async def get_coingecko_prices_by_page(cls, vs_currency: str, page_no: int) -> Dict[str, Decimal]: + results = {} + client = await cls._http_client() + async with client.request("GET", cls.coingecko_usd_price_url.format(vs_currency, page_no)) as resp: + records = await resp.json() + for record in records: + pair = f'{record["symbol"].upper()}-{vs_currency.upper()}' + if record["current_price"]: + results[pair] = Decimal(str(record["current_price"])) + return results + + async def start_network(self): + await self.stop_network() + self._fetch_price_task = safe_ensure_future(self.fetch_price_loop()) + + async def stop_network(self): + if self._fetch_price_task is not None: + self._fetch_price_task.cancel() + self._fetch_price_task = None + + async def check_network(self) -> NetworkStatus: + try: + prices = await self.get_prices() + if not prices: + raise Exception(f"Error fetching new prices from {self.source.name}.") + except asyncio.CancelledError: + raise + except Exception: + return NetworkStatus.NOT_CONNECTED + return NetworkStatus.CONNECTED + + def start(self): + NetworkBase.start(self) + + def stop(self): + NetworkBase.stop(self) diff --git a/hummingbot/core/rate_oracle/utils.py b/hummingbot/core/rate_oracle/utils.py new file mode 100644 index 0000000000..0192705764 --- /dev/null +++ b/hummingbot/core/rate_oracle/utils.py @@ -0,0 +1,30 @@ +from typing import Dict +from decimal import Decimal + + +def find_rate(prices: Dict[str, Decimal], pair: str) -> Decimal: + ''' + Finds exchange rate for a given trading pair from a dictionary of prices + For example, given prices of {"HBOT-USDT": Decimal("100"), "AAVE-USDT": Decimal("50"), "USDT-GBP": Decimal("0.75")} + A rate for USDT-HBOT will be 1 / 100 + A rate for HBOT-AAVE will be 100 / 50 + A rate for AAVE-HBOT will be 50 / 100 + A rate for HBOT-GBP will be 100 * 0.75 + :param prices: The dictionary of trading pairs and their prices + :param pair: The trading pair + ''' + if pair in prices: + return prices[pair] + base, quote = pair.split("-") + reverse_pair = f"{quote}-{base}" + if reverse_pair in prices: + return Decimal("1") / prices[reverse_pair] + base_prices = {k: v for k, v in prices.items() if k.startswith(f"{base}-")} + for base_pair, proxy_price in base_prices.items(): + link_quote = base_pair.split("-")[1] + link_pair = f"{link_quote}-{quote}" + if link_pair in prices: + return proxy_price * prices[link_pair] + common_denom_pair = f"{quote}-{link_quote}" + if common_denom_pair in prices: + return proxy_price / prices[common_denom_pair] diff --git a/hummingbot/core/utils/estimate_fee.py b/hummingbot/core/utils/estimate_fee.py index 78823e425e..5a52568861 100644 --- a/hummingbot/core/utils/estimate_fee.py +++ b/hummingbot/core/utils/estimate_fee.py @@ -2,16 +2,11 @@ from hummingbot.core.event.events import TradeFee, TradeFeeType from hummingbot.client.config.fee_overrides_config_map import fee_overrides_config_map from hummingbot.client.settings import CONNECTOR_SETTINGS -from hummingbot.core.utils.eth_gas_station_lookup import get_gas_price, get_gas_limit def estimate_fee(exchange: str, is_maker: bool) -> TradeFee: if exchange not in CONNECTOR_SETTINGS: raise Exception(f"Invalid connector. {exchange} does not exist in CONNECTOR_SETTINGS") - use_gas = CONNECTOR_SETTINGS[exchange].use_eth_gas_lookup - if use_gas: - gas_amount = get_gas_price(in_gwei=False) * get_gas_limit(exchange) - return TradeFee(percent=0, flat_fees=[("ETH", gas_amount)]) fee_type = CONNECTOR_SETTINGS[exchange].fee_type fee_token = CONNECTOR_SETTINGS[exchange].fee_token default_fees = CONNECTOR_SETTINGS[exchange].default_fees diff --git a/hummingbot/core/utils/eth_gas_station_lookup.py b/hummingbot/core/utils/eth_gas_station_lookup.py deleted file mode 100644 index ee56502631..0000000000 --- a/hummingbot/core/utils/eth_gas_station_lookup.py +++ /dev/null @@ -1,178 +0,0 @@ -import asyncio -import requests -import logging -from typing import ( - Optional, - Dict, - Any -) -import aiohttp -from enum import Enum -from decimal import Decimal -from hummingbot.core.network_base import ( - NetworkBase, - NetworkStatus -) -from hummingbot.core.utils.async_utils import safe_ensure_future -from hummingbot.logger import HummingbotLogger -from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.client.settings import CONNECTOR_SETTINGS -from hummingbot.client.settings import GATEAWAY_CA_CERT_PATH, GATEAWAY_CLIENT_CERT_PATH, GATEAWAY_CLIENT_KEY_PATH - -ETH_GASSTATION_API_URL = "https://data-api.defipulse.com/api/v1/egs/api/ethgasAPI.json?api-key={}" - - -def get_gas_price(in_gwei: bool = True) -> Decimal: - if not global_config_map["ethgasstation_gas_enabled"].value: - gas_price = global_config_map["manual_gas_price"].value - else: - gas_price = EthGasStationLookup.get_instance().gas_price - return gas_price if in_gwei else gas_price / Decimal("1e9") - - -def get_gas_limit(connector_name: str) -> int: - gas_limit = request_gas_limit(connector_name) - return gas_limit - - -def request_gas_limit(connector_name: str) -> int: - host = global_config_map["gateway_api_host"].value - port = global_config_map["gateway_api_port"].value - balancer_max_swaps = global_config_map["balancer_max_swaps"].value - - base_url = ':'.join(['https://' + host, port]) - url = f"{base_url}/{connector_name}/gas-limit" - - ca_certs = GATEAWAY_CA_CERT_PATH - client_certs = (GATEAWAY_CLIENT_CERT_PATH, GATEAWAY_CLIENT_KEY_PATH) - params = {"maxSwaps": balancer_max_swaps} if connector_name == "balancer" else {} - response = requests.post(url, data=params, verify=ca_certs, cert=client_certs) - parsed_response = response.json() - if response.status_code != 200: - err_msg = "" - if "error" in parsed_response: - err_msg = f" Message: {parsed_response['error']}" - raise IOError(f"Error fetching data from {url}. HTTP status is {response.status}.{err_msg}") - if "error" in parsed_response: - raise Exception(f"Error: {parsed_response['error']}") - return parsed_response['gasLimit'] - - -class GasLevel(Enum): - fast = "fast" - fastest = "fastest" - safeLow = "safeLow" - average = "average" - - -class EthGasStationLookup(NetworkBase): - _egsl_logger: Optional[HummingbotLogger] = None - _shared_instance: "EthGasStationLookup" = None - - @classmethod - def get_instance(cls) -> "EthGasStationLookup": - if cls._shared_instance is None: - cls._shared_instance = EthGasStationLookup() - return cls._shared_instance - - @classmethod - def logger(cls) -> HummingbotLogger: - if cls._egsl_logger is None: - cls._egsl_logger = logging.getLogger(__name__) - return cls._egsl_logger - - def __init__(self): - super().__init__() - self._gas_prices: Dict[str, Decimal] = {} - self._gas_limits: Dict[str, Decimal] = {} - self._balancer_max_swaps: int = global_config_map["balancer_max_swaps"].value - self._async_task = None - - @property - def api_key(self): - return global_config_map["ethgasstation_api_key"].value - - @property - def gas_level(self) -> GasLevel: - return GasLevel[global_config_map["ethgasstation_gas_level"].value] - - @property - def refresh_time(self): - return global_config_map["ethgasstation_refresh_time"].value - - @property - def gas_price(self): - return self._gas_prices[self.gas_level] - - @property - def gas_limits(self): - return self._gas_limits - - @gas_limits.setter - def gas_limits(self, gas_limits: Dict[str, int]): - for key, value in gas_limits.items(): - self._gas_limits[key] = value - - @property - def balancer_max_swaps(self): - return self._balancer_max_swaps - - @balancer_max_swaps.setter - def balancer_max_swaps(self, max_swaps: int): - self._balancer_max_swaps = max_swaps - - async def gas_price_update_loop(self): - while True: - try: - url = ETH_GASSTATION_API_URL.format(self.api_key) - async with aiohttp.ClientSession() as client: - response = await client.get(url=url) - if response.status != 200: - raise IOError(f"Error fetching current gas prices. " - f"HTTP status is {response.status}.") - resp_data: Dict[str, Any] = await response.json() - for key, value in resp_data.items(): - if key in GasLevel.__members__: - self._gas_prices[GasLevel[key]] = Decimal(str(value)) / Decimal("10") - prices_str = ', '.join([k.name + ': ' + str(v) for k, v in self._gas_prices.items()]) - self.logger().info(f"Gas levels: [{prices_str}]") - for name, con_setting in CONNECTOR_SETTINGS.items(): - if con_setting.use_eth_gas_lookup: - self._gas_limits[name] = get_gas_limit(name) - self.logger().info(f"{name} Gas estimate:" - f" limit = {self._gas_limits[name]:.0f}," - f" price = {self.gas_level.name}," - f" estimated cost = {get_gas_price(False) * self._gas_limits[name]:.5f} ETH") - await asyncio.sleep(self.refresh_time) - except asyncio.CancelledError: - raise - except Exception: - self.logger().network("Unexpected error running logging task.", exc_info=True) - await asyncio.sleep(self.refresh_time) - - async def start_network(self): - self._async_task = safe_ensure_future(self.gas_price_update_loop()) - - async def stop_network(self): - if self._async_task is not None: - self._async_task.cancel() - self._async_task = None - - async def check_network(self) -> NetworkStatus: - try: - url = ETH_GASSTATION_API_URL.format(self.api_key) - async with aiohttp.ClientSession() as client: - response = await client.get(url=url) - if response.status != 200: - raise Exception(f"Error connecting to {url}. HTTP status is {response.status}.") - except asyncio.CancelledError: - raise - except Exception: - return NetworkStatus.NOT_CONNECTED - return NetworkStatus.CONNECTED - - def start(self): - NetworkBase.start(self) - - def stop(self): - NetworkBase.stop(self) diff --git a/hummingbot/core/utils/ethereum.py b/hummingbot/core/utils/ethereum.py index ff667d3a0a..e30c2b3478 100644 --- a/hummingbot/core/utils/ethereum.py +++ b/hummingbot/core/utils/ethereum.py @@ -3,7 +3,11 @@ from hexbytes import HexBytes from web3 import Web3 from web3.datastructures import AttributeDict -from typing import Dict +from typing import Dict, List +import aiohttp +from hummingbot.client.config.global_config_map import global_config_map +import itertools as it +from hummingbot.core.utils import async_ttl_cache def check_web3(ethereum_rpc_url: str) -> bool: @@ -32,3 +36,55 @@ def block_values_to_hex(block: AttributeDict) -> AttributeDict: except binascii.Error: formatted_block[key] = value return AttributeDict(formatted_block) + + +def check_transaction_exceptions(trade_data: dict) -> dict: + + exception_list = [] + + gas_limit = trade_data["gas_limit"] + # gas_price = trade_data["gas_price"] + gas_cost = trade_data["gas_cost"] + amount = trade_data["amount"] + side = trade_data["side"] + base = trade_data["base"] + quote = trade_data["quote"] + balances = trade_data["balances"] + allowances = trade_data["allowances"] + swaps_message = f"Total swaps: {trade_data['swaps']}" if "swaps" in trade_data.keys() else '' + + eth_balance = balances["ETH"] + + # check for sufficient gas + if eth_balance < gas_cost: + exception_list.append(f"Insufficient ETH balance to cover gas:" + f" Balance: {eth_balance}. Est. gas cost: {gas_cost}. {swaps_message}") + + trade_token = base if side == "side" else quote + trade_allowance = allowances[trade_token] + + # check for gas limit set to low + gas_limit_threshold = 21000 + if gas_limit < gas_limit_threshold: + exception_list.append(f"Gas limit {gas_limit} below recommended {gas_limit_threshold} threshold.") + + # check for insufficient token allowance + if allowances[trade_token] < amount: + exception_list.append(f"Insufficient {trade_token} allowance {trade_allowance}. Amount to trade: {amount}") + + return exception_list + + +@async_ttl_cache(ttl=30) +async def fetch_trading_pairs() -> List[str]: + token_list_url = global_config_map.get("ethereum_token_list_url").value + tokens = set() + async with aiohttp.ClientSession() as client: + resp = await client.get(token_list_url) + resp_json = await resp.json() + for token in resp_json["tokens"]: + tokens.add(token["symbol"]) + trading_pairs = [] + for base, quote in it.permutations(tokens, 2): + trading_pairs.append(f"{base}-{quote}") + return trading_pairs diff --git a/hummingbot/core/utils/market_price.py b/hummingbot/core/utils/market_price.py index 1fa0c66ab7..385329dc5b 100644 --- a/hummingbot/core/utils/market_price.py +++ b/hummingbot/core/utils/market_price.py @@ -1,17 +1,8 @@ from typing import Optional, Dict from decimal import Decimal import importlib -from hummingbot.core.utils import async_ttl_cache -from hummingbot.client.settings import ALL_CONNECTORS, CONNECTOR_SETTINGS, ConnectorType +from hummingbot.client.settings import CONNECTOR_SETTINGS, ConnectorType from hummingbot.connector.exchange.binance.binance_api_order_book_data_source import BinanceAPIOrderBookDataSource -from hummingbot.connector.exchange.binance.binance_utils import USD_QUOTES - - -async def usd_value(token: str, amount: Decimal) -> Optional[Decimal]: - if token in USD_QUOTES: - return amount - usd_values = await token_usd_values() - return usd_values.get(token, 0) * amount async def get_binance_mid_price(trading_pair: str) -> Dict[str, Decimal]: @@ -20,51 +11,6 @@ async def get_binance_mid_price(trading_pair: str) -> Dict[str, Decimal]: return prices.get(trading_pair, None) -@async_ttl_cache(ttl=5, maxsize=100) -async def token_usd_values() -> Dict[str, Decimal]: - prices = await BinanceAPIOrderBookDataSource.get_all_mid_prices() - prices = {k: v for k, v in prices.items() if k is not None} - tokens = {t.split("-")[0] for t in prices} - ret_val = {} - for token in tokens: - token_usd_pairs = [t for t in prices if t.split("-")[0] == token and t.split("-")[1] in USD_QUOTES] - if token_usd_pairs: - ret_val[token] = max([prices[usd_pair] for usd_pair in token_usd_pairs]) - else: - token_any_pairs = [t for t, price in prices.items() if t.split("-")[0] == token and price > 0] - if not token_any_pairs: - continue - quote = token_any_pairs[0].split("-")[1] - quote_usds = [t for t in prices if t.split("-")[0] == quote and t.split("-")[1] in USD_QUOTES] - if quote_usds: - price = prices[token_any_pairs[0]] * prices[quote_usds[0]] - ret_val[token] = price - return ret_val - - -def get_mid_price(exchange: str, trading_pair: str) -> Optional[Decimal]: - - mid_price = None - for connector_type, connectors in ALL_CONNECTORS.items(): - if exchange in connectors: - try: - module_name = f"{exchange}_api_order_book_data_source" - class_name = "".join([o.capitalize() for o in exchange.split("_")]) + "APIOrderBookDataSource" - module_path = f"hummingbot.connector.{connector_type}.{exchange}.{module_name}" - module = getattr(importlib.import_module(module_path), class_name) - mid_price = module.get_mid_price(trading_pair) - except Exception: - pass - if mid_price is None: - module_name = "binance_api_order_book_data_source" - class_name = "BinanceAPIOrderBookDataSource" - module_path = f"hummingbot.connector.exchange.binance.{module_name}" - module = getattr(importlib.import_module(module_path), class_name) - mid_price = module.get_mid_price(trading_pair) - - return mid_price - - async def get_last_price(exchange: str, trading_pair: str) -> Optional[Decimal]: if exchange in CONNECTOR_SETTINGS: conn_setting = CONNECTOR_SETTINGS[exchange] diff --git a/hummingbot/core/utils/trading_pair_fetcher.py b/hummingbot/core/utils/trading_pair_fetcher.py index 1da79fa514..ae79aefe9a 100644 --- a/hummingbot/core/utils/trading_pair_fetcher.py +++ b/hummingbot/core/utils/trading_pair_fetcher.py @@ -4,11 +4,11 @@ Any, Optional, ) -from hummingbot.core.utils.async_utils import safe_gather from hummingbot.logger import HummingbotLogger from hummingbot.client.settings import CONNECTOR_SETTINGS, ConnectorType import logging import asyncio +import requests from .async_utils import safe_ensure_future @@ -35,8 +35,6 @@ def __init__(self): safe_ensure_future(self.fetch_all()) async def fetch_all(self): - tasks = [] - fetched_connectors = [] for conn_setting in CONNECTOR_SETTINGS.values(): module_name = f"{conn_setting.base_name()}_connector" if conn_setting.type is ConnectorType.Connector \ else f"{conn_setting.base_name()}_api_order_book_data_source" @@ -48,8 +46,8 @@ async def fetch_all(self): module = getattr(importlib.import_module(module_path), class_name) args = {} args = conn_setting.add_domain_parameter(args) - tasks.append(asyncio.wait_for(asyncio.shield(module.fetch_trading_pairs(**args)), timeout=3)) - fetched_connectors.append(conn_setting.name) + safe_ensure_future(self.call_fetch_pairs(module.fetch_trading_pairs(**args), conn_setting.name)) + results = await safe_gather(*tasks, return_exceptions=True) self.trading_pairs = dict(zip(fetched_connectors, results)) @@ -57,4 +55,5 @@ async def fetch_all(self): for connector, result in self.trading_pairs.items(): if isinstance(result, asyncio.TimeoutError): self.trading_pairs[connector] = [] - self.ready = True + + self.ready = True \ No newline at end of file diff --git a/hummingbot/strategy/amm_arb/amm_arb.py b/hummingbot/strategy/amm_arb/amm_arb.py index b96af808fc..381cd991cc 100644 --- a/hummingbot/strategy/amm_arb/amm_arb.py +++ b/hummingbot/strategy/amm_arb/amm_arb.py @@ -253,13 +253,18 @@ async def format_status(self) -> str: market, trading_pair, base_asset, quote_asset = market_info buy_price = await market.get_quote_price(trading_pair, True, self._order_amount) sell_price = await market.get_quote_price(trading_pair, False, self._order_amount) - mid_price = (buy_price + sell_price) / 2 + + # check for unavailable price data + buy_price = Decimal(str(buy_price)) if buy_price is not None else '-' + sell_price = Decimal(str(sell_price)) if sell_price is not None else '-' + mid_price = ((buy_price + sell_price) / 2) if '-' not in [buy_price, sell_price] else '-' + data.append([ market.display_name, trading_pair, - float(sell_price), - float(buy_price), - float(mid_price) + sell_price, + buy_price, + mid_price ]) markets_df = pd.DataFrame(data=data, columns=columns) lines = [] @@ -335,7 +340,7 @@ async def quote_in_eth_rate_fetch_loop(self): self._market_2_quote_eth_rate = await self.request_rate_in_eth(self._market_info_2.quote_asset) self.logger().warning(f"Estimate conversion rate - " f"{self._market_info_2.quote_asset}:ETH = {self._market_2_quote_eth_rate} ") - await asyncio.sleep(60 * 5) + await asyncio.sleep(60 * 1) except asyncio.CancelledError: raise except Exception as e: @@ -348,4 +353,5 @@ async def quote_in_eth_rate_fetch_loop(self): async def request_rate_in_eth(self, quote: str) -> int: if self._uniswap is None: self._uniswap = UniswapConnector([f"{quote}-WETH"], "", None) + await self._uniswap.initiate_pool() # initiate to cache swap pool return await self._uniswap.get_quote_price(f"{quote}-WETH", True, 1) diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py index 2d8f2438a9..65eaaa0554 100644 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py @@ -48,19 +48,19 @@ def validate_taker_market_trading_pair(value: str) -> Optional[str]: return validate_market_trading_pair(taker_market, value) -def order_amount_prompt() -> str: +async def order_amount_prompt() -> str: maker_exchange = cross_exchange_market_making_config_map["maker_market"].value trading_pair = cross_exchange_market_making_config_map["maker_market_trading_pair"].value base_asset, quote_asset = trading_pair.split("-") - min_amount = minimum_order_amount(maker_exchange, trading_pair) + min_amount = await minimum_order_amount(maker_exchange, trading_pair) return f"What is the amount of {base_asset} per order? (minimum {min_amount}) >>> " -def validate_order_amount(value: str) -> Optional[str]: +async def validate_order_amount(value: str) -> Optional[str]: try: maker_exchange = cross_exchange_market_making_config_map.get("maker_market").value trading_pair = cross_exchange_market_making_config_map["maker_market_trading_pair"].value - min_amount = minimum_order_amount(maker_exchange, trading_pair) + min_amount = await minimum_order_amount(maker_exchange, trading_pair) if Decimal(value) < min_amount: return f"Order amount must be at least {min_amount}." except Exception: diff --git a/hummingbot/strategy/liquidity_mining/liquidity_mining.py b/hummingbot/strategy/liquidity_mining/liquidity_mining.py index 1b262376c2..b0de1f8cf7 100644 --- a/hummingbot/strategy/liquidity_mining/liquidity_mining.py +++ b/hummingbot/strategy/liquidity_mining/liquidity_mining.py @@ -15,11 +15,12 @@ from hummingbot.core.event.events import OrderType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.utils.estimate_fee import estimate_fee -from hummingbot.core.utils.market_price import usd_value from hummingbot.strategy.pure_market_making.inventory_skew_calculator import ( calculate_bid_ask_ratios_from_base_asset_ratio ) from hummingbot.connector.parrot import get_campaign_summary +from hummingbot.core.rate_oracle.rate_oracle import RateOracle + NaN = float("nan") s_decimal_zero = Decimal(0) s_decimal_nan = Decimal("NaN") @@ -192,15 +193,16 @@ def market_status_df(self) -> pd.DataFrame: async def miner_status_df(self) -> pd.DataFrame: data = [] + g_sym = RateOracle.global_token_symbol columns = ["Market", "Payout", "Reward/wk", "Liquidity", "Yield/yr", "Max spread"] campaigns = await get_campaign_summary(self._exchange.display_name, list(self._market_infos.keys())) for market, campaign in campaigns.items(): - reward_usd = await usd_value(campaign.payout_asset, campaign.reward_per_wk) + reward = await RateOracle.global_value(campaign.payout_asset, campaign.reward_per_wk) data.append([ market, campaign.payout_asset, - f"${reward_usd:.0f}", - f"${campaign.liquidity_usd:.0f}", + f"{g_sym}{reward:.0f}", + f"{g_sym}{campaign.liquidity_usd:.0f}", f"{campaign.apy:.2%}", f"{campaign.spread_max:.2%}%" ]) diff --git a/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py b/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py index 1040ef9c53..978b93091e 100644 --- a/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py +++ b/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py @@ -39,19 +39,19 @@ def validate_derivative_position_mode(value: str) -> Optional[str]: return "Position mode can either be One-way or Hedge mode" -def order_amount_prompt() -> str: +async def order_amount_prompt() -> str: derivative = perpetual_market_making_config_map["derivative"].value trading_pair = perpetual_market_making_config_map["market"].value base_asset, quote_asset = trading_pair.split("-") - min_amount = minimum_order_amount(derivative, trading_pair) + min_amount = await minimum_order_amount(derivative, trading_pair) return f"What is the amount of {base_asset} per order? (minimum {min_amount}) >>> " -def validate_order_amount(value: str) -> Optional[str]: +async def validate_order_amount(value: str) -> Optional[str]: try: derivative = perpetual_market_making_config_map["derivative"].value trading_pair = perpetual_market_making_config_map["market"].value - min_amount = minimum_order_amount(derivative, trading_pair) + min_amount = await minimum_order_amount(derivative, trading_pair) if Decimal(value) < min_amount: return f"Order amount must be at least {min_amount}." except Exception: diff --git a/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py b/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py index 7f8065b698..1501d16c6c 100644 --- a/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py +++ b/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py @@ -35,19 +35,19 @@ def validate_exchange_trading_pair(value: str) -> Optional[str]: return validate_market_trading_pair(exchange, value) -def order_amount_prompt() -> str: +async def order_amount_prompt() -> str: exchange = pure_market_making_config_map["exchange"].value trading_pair = pure_market_making_config_map["market"].value base_asset, quote_asset = trading_pair.split("-") - min_amount = minimum_order_amount(exchange, trading_pair) + min_amount = await minimum_order_amount(exchange, trading_pair) return f"What is the amount of {base_asset} per order? (minimum {min_amount}) >>> " -def validate_order_amount(value: str) -> Optional[str]: +async def validate_order_amount(value: str) -> Optional[str]: try: exchange = pure_market_making_config_map["exchange"].value trading_pair = pure_market_making_config_map["market"].value - min_amount = minimum_order_amount(exchange, trading_pair) + min_amount = await minimum_order_amount(exchange, trading_pair) if Decimal(value) < min_amount: return f"Order amount must be at least {min_amount}." except Exception: diff --git a/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml b/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml index c4f1e53d48..3aff911e85 100644 --- a/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml +++ b/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml @@ -44,8 +44,9 @@ kraken_taker_fee: dolomite_maker_fee_amount: dolomite_taker_fee_amount: -eterbase_maker_fee: -eterbase_taker_fee: + +hitbtc_maker_fee: +hitbtc_taker_fee: loopring_maker_fee: loopring_taker_fee: @@ -71,9 +72,15 @@ okex_taker_fee: balancer_maker_fee_amount: balancer_taker_fee_amount: +uniswap_maker_fee_amount: +uniswap_taker_fee_amount: + bitmax_maker_fee: bitmax_taker_fee: +ascend_ex_maker_fee: +ascend_ex_taker_fee: + probit_maker_fee: probit_taker_fee: diff --git a/hummingbot/templates/conf_global_TEMPLATE.yml b/hummingbot/templates/conf_global_TEMPLATE.yml index ebbb32196e..508af5d1ff 100644 --- a/hummingbot/templates/conf_global_TEMPLATE.yml +++ b/hummingbot/templates/conf_global_TEMPLATE.yml @@ -3,7 +3,7 @@ ################################# # For more detailed information: https://docs.hummingbot.io -template_version: 18 +template_version: 20 # Exchange configs bamboo_relay_use_coordinator: false @@ -58,9 +58,9 @@ kraken_secret_key: null crypto_com_api_key: null crypto_com_secret_key: null -eterbase_api_key: null -eterbase_secret_key: null -eterbase_account: null + +hitbtc_api_key: null +hitbtc_secret_key: null bitfinex_api_key: null bitfinex_secret_key: null @@ -69,8 +69,8 @@ okex_api_key: null okex_secret_key: null okex_passphrase: null -bitmax_api_key: null -bitmax_secret_key: null +ascend_ex_api_key: null +ascend_ex_secret_key: null celo_address: null celo_password: null @@ -78,6 +78,9 @@ celo_password: null terra_wallet_address: null terra_wallet_seeds: null +digifinex_api_key: null +digifinex_secret_key: null + balancer_max_swaps: 4 probit_api_key: null @@ -91,13 +94,16 @@ ethereum_wallet: null ethereum_rpc_url: null ethereum_rpc_ws_url: null ethereum_chain_name: MAIN_NET -ethereum_token_list_url: null +ethereum_token_list_url: https://wispy-bird-88a7.uniswap.workers.dev/?url=http://tokens.1inch.eth.link # Kill switch kill_switch_enabled: null # The rate of performance at which you would want the bot to stop trading (-20 = 20%) kill_switch_rate: null +# What to auto-fill in the prompt after each import command (start/config) +autofill_import: null + # DEX active order cancellation 0x_active_cancels: false @@ -169,14 +175,7 @@ balance_asset_limit: # Fixed gas price (in Gwei) for Ethereum transactions manual_gas_price: -# To enable gas price lookup (true/false) -ethgasstation_gas_enabled: -# API key for defipulse.com gas station API -ethgasstation_api_key: -# Gas level you want to use for Ethereum transactions (fast, fastest, safeLow, average) -ethgasstation_gas_level: -# Refresh time for Ethereum gas price lookups (in seconds) -ethgasstation_refresh_time: + # Gateway API Configurations # default host to only use localhost # Port need to match the final installation port for Gateway @@ -189,3 +188,9 @@ heartbeat_enabled: heartbeat_interval_min: # a list of binance markets (for trades/pnl reporting) separated by ',' e.g. RLC-USDT,RLC-BTC binance_markets: + +rate_oracle_source: + +global_token: + +global_token_symbol: \ No newline at end of file diff --git a/hummingbot/user/user_balances.py b/hummingbot/user/user_balances.py index b9e6e313c2..445c00b963 100644 --- a/hummingbot/user/user_balances.py +++ b/hummingbot/user/user_balances.py @@ -1,4 +1,4 @@ -from hummingbot.core.utils.market_price import get_mid_price +from hummingbot.core.utils.market_price import get_last_price from hummingbot.client.settings import CONNECTOR_SETTINGS from hummingbot.client.config.security import Security from hummingbot.client.config.config_helpers import get_connector_class, get_eth_wallet_private_key @@ -149,12 +149,12 @@ def validate_ethereum_wallet() -> Optional[str]: return None @staticmethod - def base_amount_ratio(exchange, trading_pair, balances) -> Optional[Decimal]: + async def base_amount_ratio(exchange, trading_pair, balances) -> Optional[Decimal]: try: base, quote = trading_pair.split("-") base_amount = balances.get(base, 0) quote_amount = balances.get(quote, 0) - price = get_mid_price(exchange, trading_pair) + price = await get_last_price(exchange, trading_pair) total_value = base_amount + (quote_amount / price) return None if total_value <= 0 else base_amount / total_value except Exception: diff --git a/installation/docker-commands/create-gateway.sh b/installation/docker-commands/create-gateway.sh index c6db0bae8c..49fc98579c 100755 --- a/installation/docker-commands/create-gateway.sh +++ b/installation/docker-commands/create-gateway.sh @@ -40,7 +40,7 @@ else else echo "‼️ hummingbot_conf & hummingbot_certs directory missing from path $FOLDER" prompt_hummingbot_data_path - fi + fi if [[ -f "$FOLDER/hummingbot_certs/server_cert.pem" && -f "$FOLDER/hummingbot_certs/server_key.pem" && -f "$FOLDER/hummingbot_certs/ca_cert.pem" ]]; then echo @@ -79,70 +79,208 @@ do then HUMMINGBOT_INSTANCE_ID="$(echo -e "${value}" | tr -d '[:space:]')" fi - # chain - if [ "$key" == "ethereum_chain_name" ] + # +done < "$GLOBAL_CONFIG" +} +read_global_config + +# prompt to setup balancer, uniswap +prompt_ethereum_setup () { + read -p " Do you want to setup Balancer or Uniswap? [Y/N] (default \"Y\") >>> " PROCEED + if [[ "$PROCEED" == "Y" || "$PROCEED" == "y" || "$PROCEED" == "" ]] then - ETHEREUM_CHAIN="$(echo -e "${value}" | tr -d '[:space:]')" - # subgraph url - if [[ "$ETHEREUM_CHAIN" == "MAIN_NET" || "$ETHEREUM_CHAIN" == "main_net" || "$ETHEREUM_CHAIN" == "MAINNET" || "$ETHEREUM_CHAIN" == "mainnet" ]] + ETHEREUM_SETUP=true + echo + read -p " Enter Ethereum chain you want to use [mainnet/kovan] (default = \"mainnet\") >>> " ETHEREUM_CHAIN + # chain selection + if [ "$ETHEREUM_CHAIN" == "" ] + then + ETHEREUM_CHAIN="mainnet" + fi + if [[ "$ETHEREUM_CHAIN" != "mainnet" && "$ETHEREUM_CHAIN" != "kovan" ]] + then + echo "‼️ ERROR. Unsupported chains (mainnet/kovan). " + prompt_ethereum_setup + fi + # set subgraph url, exchange_proxy + if [[ "$ETHEREUM_CHAIN" == "mainnet" ]] then ETHEREUM_CHAIN="mainnet" REACT_APP_SUBGRAPH_URL="https://api.thegraph.com/subgraphs/name/balancer-labs/balancer" EXCHANGE_PROXY="0x3E66B66Fd1d0b02fDa6C811Da9E0547970DB2f21" else - ETHEREUM_CHAIN="kovan" - REACT_APP_SUBGRAPH_URL="https://api.thegraph.com/subgraphs/name/balancer-labs/balancer-kovan" - EXCHANGE_PROXY="0x4e67bf5bD28Dd4b570FBAFe11D0633eCbA2754Ec" + if [[ "$ETHEREUM_CHAIN" == "kovan" ]] + then + REACT_APP_SUBGRAPH_URL="https://api.thegraph.com/subgraphs/name/balancer-labs/balancer-kovan" + EXCHANGE_PROXY="0x4e67bf5bD28Dd4b570FBAFe11D0633eCbA2754Ec" + fi fi fi - # ethereum rpc url - if [ "$key" == "ethereum_rpc_url" ] +} +prompt_ethereum_setup + +# prompt to ethereum rpc +prompt_ethereum_rpc_setup () { + if [ "$ETHEREUM_RPC_URL" == "" ] then - ETHEREUM_RPC_URL="$(echo -e "${value}" | tr -d '[:space:]')" + read -p " Enter the Ethereum RPC node URL to connect to >>> " ETHEREUM_RPC_URL + if [ "$ETHEREUM_RPC_URL" == "" ] + then + prompt_ethereum_rpc_setup + fi + else + read -p " Use the this Ethereum RPC node ($ETHEREUM_RPC_URL) setup in Hummingbot client? [Y/N] (default = \"Y\") >>> " PROCEED + if [[ "$PROCEED" == "Y" || "$PROCEED" == "y" || "$PROCEED" == "" ]] + then + echo + else + ETHEREUM_RPC_URL="" + prompt_ethereum_rpc_setup + fi fi -done < "$GLOBAL_CONFIG" } -read_global_config +prompt_ethereum_rpc_setup -# prompt to setup balancer, uniswap -prompt_ethereum_setup () { - read -p " Do you want to setup Balancer/Uniswap/Perpetual Finance? [Y/N] >>> " PROCEED - if [[ "$PROCEED" == "Y" || "$PROCEED" == "y" ]] +# prompt to setup ethereum token list +prompt_token_list_source () { + echo + echo " Enter the token list url available at https://tokenlists.org/" + read -p " (default = \"https://wispy-bird-88a7.uniswap.workers.dev/?url=http://tokens.1inch.eth.link\") >>> " ETHEREUM_TOKEN_LIST_URL + if [ "$ETHEREUM_TOKEN_LIST_URL" == "" ] then + echo echo "ℹ️ Retrieving config from Hummingbot config file ... " ETHEREUM_SETUP=true + ETHEREUM_TOKEN_LIST_URL=https://wispy-bird-88a7.uniswap.workers.dev/?url=http://tokens.1inch.eth.link + fi +} +prompt_token_list_source + +# prompt to setup eth gas level +prompt_eth_gasstation_gas_level () { + echo + read -p " Enter gas level you want to use for Ethereum transactions (fast, fastest, safeLow, average) (default = \"fast\") >>> " ETH_GAS_STATION_GAS_LEVEL + if [ "$ETH_GAS_STATION_GAS_LEVEL" == "" ] + then + ETH_GAS_STATION_GAS_LEVEL=fast + else + if [[ "$ETH_GAS_STATION_GAS_LEVEL" != "fast" && "$ETH_GAS_STATION_GAS_LEVEL" != "fastest" && "$ETH_GAS_STATION_GAS_LEVEL" != "safeLow" && "$ETH_GAS_STATION_GAS_LEVEL" != "safelow" && "$ETH_GAS_STATION_GAS_LEVEL" != "average" ]] + then + prompt_eth_gasstation_gas_level + fi + fi +} + +# prompt to setup eth gas station +prompt_eth_gasstation_setup () { + echo + read -p " Enable dynamic Ethereum gas price lookup? [Y/N] (default = \"Y\") >>> " PROCEED + if [[ "$PROCEED" == "Y" || "$PROCEED" == "y" || "$PROCEED" == "" ]] + then + ENABLE_ETH_GAS_STATION=true + read -p " Enter API key for Eth Gas Station (https://ethgasstation.info/) >>> " ETH_GAS_STATION_API_KEY + if [ "$ETH_GAS_STATION_API_KEY" == "" ] + then + prompt_eth_gasstation_setup + else + # set gas level + prompt_eth_gasstation_gas_level + + # set refresh interval + read -p " Enter refresh time for Ethereum gas price lookup (in seconds) (default = \"120\") >>> " ETH_GAS_STATION_REFRESH_TIME + if [ "$ETH_GAS_STATION_REFRESH_TIME" == "" ] + then + ETH_GAS_STATION_REFRESH_TIME=120 + fi + fi + else + if [[ "$PROCEED" == "N" || "$PROCEED" == "n" ]] + then + ENABLE_ETH_GAS_STATION=false + # set manual gas price + read -p " Enter fixed gas price (in Gwei) you want to use for Ethereum transactions (default = \"100\") >>> " MANUAL_GAS_PRICE + if [ "$MANUAL_GAS_PRICE" == "" ] + then + MANUAL_GAS_PRICE=100 + fi + else + prompt_eth_gasstation_setup + fi + fi + echo +} +prompt_eth_gasstation_setup + +prompt_balancer_setup () { + # Ask the user for the Balancer specific settings + echo "ℹ️ Balancer setting " + read -p " Enter the maximum Balancer swap pool (default = \"4\") >>> " BALANCER_MAX_SWAPS + if [ "$BALANCER_MAX_SWAPS" == "" ] + then + BALANCER_MAX_SWAPS="4" echo fi } -prompt_ethereum_setup -# Ask the user for ethereum network -prompt_terra_network () { -read -p " Enter Terra chain you want to use [mainnet/testnet] (default = \"mainnet\") >>> " TERRA -# chain selection -if [ "$TERRA" == "" ] -then - TERRA="mainnet" -fi -if [[ "$TERRA" != "mainnet" && "$TERRA" != "testnet" ]] -then - echo "‼️ ERROR. Unsupported chains (mainnet/testnet). " - prompt_terra_network -fi -# setup chain params -if [[ "$TERRA" == "mainnet" ]] -then - TERRA_LCD_URL="https://lcd.terra.dev" - TERRA_CHAIN="columbus-4" -elif [ "$TERRA" == "testnet" ] +prompt_uniswap_setup () { + # Ask the user for the Uniswap specific settings + echo "ℹ️ Uniswap setting " + read -p " Enter the allowed slippage for swap transactions (default = \"1.5\") >>> " UNISWAP_SLIPPAGE + if [ "$UNISWAP_SLIPPAGE" == "" ] + then + UNISWAP_SLIPPAGE="1.5" + echo + fi +} + +if [[ "$ETHEREUM_SETUP" == true ]] then - TERRA_LCD_URL="https://tequila-lcd.terra.dev" - TERRA_CHAIN="tequila-0004" + prompt_balancer_setup + prompt_uniswap_setup fi + +prompt_xdai_setup () { + # Ask the user for the Uniswap specific settings + echo "ℹ️ XDAI setting " + read -p " Enter preferred XDAI rpc provider (default = \"https://rpc.xdaichain.com\") >>> " XDAI_PROVIDER + if [ "$XDAI_PROVIDER" == "" ] + then + XDAI_PROVIDER="https://rpc.xdaichain.com" + echo + fi } +prompt_xdai_setup + +# Ask the user for ethereum network +prompt_terra_network () { + echo + read -p " Enter Terra chain you want to use [mainnet/testnet] (default = \"mainnet\") >>> " TERRA + # chain selection + if [ "$TERRA" == "" ] + then + TERRA="mainnet" + fi + if [[ "$TERRA" != "mainnet" && "$TERRA" != "testnet" ]] + then + echo "‼️ ERROR. Unsupported chains (mainnet/testnet). " + prompt_terra_network + fi + # setup chain params + if [[ "$TERRA" == "mainnet" ]] + then + TERRA_LCD_URL="https://lcd.terra.dev" + TERRA_CHAIN="columbus-4" + elif [ "$TERRA" == "testnet" ] + then + TERRA_LCD_URL="https://tequila-lcd.terra.dev" + TERRA_CHAIN="tequila-0004" + fi +} + prompt_terra_setup () { - read -p " Do you want to setup Terra? [Y/N] >>> " PROCEED - if [[ "$PROCEED" == "Y" || "$PROCEED" == "y" ]] + echo + read -p " Do you want to setup Terra? [Y/N] (default \"Y\") >>> " PROCEED + if [[ "$PROCEED" == "Y" || "$PROCEED" == "y" || "$PROCEED" == "" ]] then TERRA_SETUP=true prompt_terra_network @@ -202,9 +340,17 @@ echo printf "%30s %5s\n" "Hummingbot Instance ID:" "$HUMMINGBOT_INSTANCE_ID" printf "%30s %5s\n" "Ethereum Chain:" "$ETHEREUM_CHAIN" printf "%30s %5s\n" "Ethereum RPC URL:" "$ETHEREUM_RPC_URL" +printf "%30s %5s\n" "Ethereum Token List URL:" "$ETHEREUM_TOKEN_LIST_URL" +printf "%30s %5s\n" "Manual Gas Price:" "$MANUAL_GAS_PRICE" +printf "%30s %5s\n" "Enable Eth Gas Station:" "$ENABLE_ETH_GAS_STATION" +printf "%30s %5s\n" "Eth Gas Station API:" "$ETH_GAS_STATION_API_KEY" +printf "%30s %5s\n" "Eth Gas Station Level:" "$ETH_GAS_STATION_GAS_LEVEL" +printf "%30s %5s\n" "Eth Gas Station Refresh Interval:" "$ETH_GAS_STATION_REFRESH_TIME" printf "%30s %5s\n" "Balancer Subgraph:" "$REACT_APP_SUBGRAPH_URL" printf "%30s %5s\n" "Balancer Exchange Proxy:" "$EXCHANGE_PROXY" +printf "%30s %5s\n" "Balancer Max Swaps:" "$BALANCER_MAX_SWAPS" printf "%30s %5s\n" "Uniswap Router:" "$UNISWAP_ROUTER" +printf "%30s %5s\n" "Uniswap Allowed Slippage:" "$UNISWAP_SLIPPAGE" printf "%30s %5s\n" "Terra Chain:" "$TERRA" printf "%30s %5s\n" "Gateway Log Path:" "$LOG_PATH" printf "%30s %5s\n" "Gateway Cert Path:" "$CERT_PATH" @@ -220,13 +366,46 @@ echo "NODE_ENV=prod" >> $ENV_FILE echo "PORT=$PORT" >> $ENV_FILE echo "" >> $ENV_FILE echo "HUMMINGBOT_INSTANCE_ID=$HUMMINGBOT_INSTANCE_ID" >> $ENV_FILE + +# ethereum config +echo "" >> $ENV_FILE +echo "# Ethereum Settings" >> $ENV_FILE echo "ETHEREUM_CHAIN=$ETHEREUM_CHAIN" >> $ENV_FILE echo "ETHEREUM_RPC_URL=$ETHEREUM_RPC_URL" >> $ENV_FILE +echo "ETHEREUM_TOKEN_LIST_URL=$ETHEREUM_TOKEN_LIST_URL" >> $ENV_FILE +echo "" >> $ENV_FILE +echo "ENABLE_ETH_GAS_STATION=$ENABLE_ETH_GAS_STATION" >> $ENV_FILE +echo "ETH_GAS_STATION_API_KEY=$ETH_GAS_STATION_API_KEY" >> $ENV_FILE +echo "ETH_GAS_STATION_GAS_LEVEL=$ETH_GAS_STATION_GAS_LEVEL" >> $ENV_FILE +echo "ETH_GAS_STATION_REFRESH_TIME=$ETH_GAS_STATION_REFRESH_TIME" >> $ENV_FILE +echo "MANUAL_GAS_PRICE=$MANUAL_GAS_PRICE" >> $ENV_FILE + +# balancer config +echo "" >> $ENV_FILE +echo "# Balancer Settings" >> $ENV_FILE echo "REACT_APP_SUBGRAPH_URL=$REACT_APP_SUBGRAPH_URL" >> $ENV_FILE # must used "REACT_APP_SUBGRAPH_URL" for balancer-sor echo "EXCHANGE_PROXY=$EXCHANGE_PROXY" >> $ENV_FILE +echo "BALANCER_MAX_SWAPS=$BALANCER_MAX_SWAPS" >> $ENV_FILE + +# uniswap config +echo "" >> $ENV_FILE +echo "# Uniswap Settings" >> $ENV_FILE echo "UNISWAP_ROUTER=$UNISWAP_ROUTER" >> $ENV_FILE +echo "UNISWAP_ALLOWED_SLIPPAGE=$UNISWAP_SLIPPAGE" >> $ENV_FILE +echo "UNISWAP_NO_RESERVE_CHECK_INTERVAL=300000" >> $ENV_FILE +echo "UNISWAP_PAIRS_CACHE_TIME=1000" >> $ENV_FILE + +# terra config +echo "" >> $ENV_FILE +echo "# Terra Settings" >> $ENV_FILE echo "TERRA_LCD_URL=$TERRA_LCD_URL" >> $ENV_FILE echo "TERRA_CHAIN=$TERRA_CHAIN" >> $ENV_FILE + +# perpeptual finance config +echo "" >> $ENV_FILE +echo "# Perpeptual Settings" >> $ENV_FILE +echo "XDAI_PROVIDER=$XDAI_PROVIDER" >> $ENV_FILE + echo "" >> $ENV_FILE prompt_proceed () { @@ -234,7 +413,12 @@ prompt_proceed () { read -p " Do you want to proceed with installation? [Y/N] >>> " PROCEED if [ "$PROCEED" == "" ] then - PROCEED="Y" + prompt_proceed + else + if [[ "$PROCEED" != "Y" && "$PROCEED" != "y" ]] + then + PROCEED="N" + fi fi } diff --git a/setup.py b/setup.py index 6cd0dd9002..6ad90e8f60 100755 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ def main(): "hummingbot.connector.connector.balancer", "hummingbot.connector.connector.terra", "hummingbot.connector.exchange", + "hummingbot.connector.exchange.ascend_ex", "hummingbot.connector.exchange.binance", "hummingbot.connector.exchange.bitfinex", "hummingbot.connector.exchange.bittrex", @@ -56,7 +57,7 @@ def main(): "hummingbot.connector.exchange.dolomite", "hummingbot.connector.exchange.eterbase", "hummingbot.connector.exchange.beaxy", - "hummingbot.connector.exchange.bitmax", + "hummingbot.connector.exchange.hitbtc", "hummingbot.connector.derivative", "hummingbot.connector.derivative.binance_perpetual", "hummingbot.script", diff --git a/setup/requirements-arm.txt b/setup/requirements-arm.txt index 93ed82682e..d50a0073af 100644 --- a/setup/requirements-arm.txt +++ b/setup/requirements-arm.txt @@ -22,7 +22,7 @@ mypy-extensions==0.4.3 python-telegram-bot==12.8 python-binance==0.7.5 pandas==1.1.0 -aiohttp==3.6.2 +aiohttp==3.7.4 bitstring==3.1.7 pyblake2==1.1.2 pysha3==1.0.2 diff --git a/test/connector/connector/balancer/test_balancer_connector.py b/test/connector/connector/balancer/test_balancer_connector.py index 151d0e403e..66ad53df97 100644 --- a/test/connector/connector/balancer/test_balancer_connector.py +++ b/test/connector/connector/balancer/test_balancer_connector.py @@ -28,8 +28,7 @@ global_config_map['gateway_api_host'].value = "localhost" global_config_map['gateway_api_port'].value = 5000 -global_config_map['ethgasstation_gas_enabled'].value = False -global_config_map['manual_gas_price'].value = 50 +global_config_map['ethereum_token_list_url'].value = "https://defi.cmc.eth.link" global_config_map.get("ethereum_chain_name").value = "kovan" trading_pair = "WETH-DAI" @@ -96,6 +95,14 @@ def setUp(self): for event_tag in self.events: self.connector.add_listener(event_tag, self.event_logger) + def test_fetch_trading_pairs(self): + asyncio.get_event_loop().run_until_complete(self._test_fetch_trading_pairs()) + + async def _test_fetch_trading_pairs(self): + pairs = await BalancerConnector.fetch_trading_pairs() + print(pairs) + self.assertGreater(len(pairs), 0) + def test_update_balances(self): all_bals = self.connector.get_all_balances() for token, bal in all_bals.items(): diff --git a/test/connector/exchange/bitmax/.gitignore b/test/connector/exchange/ascend_ex/.gitignore similarity index 100% rename from test/connector/exchange/bitmax/.gitignore rename to test/connector/exchange/ascend_ex/.gitignore diff --git a/test/connector/exchange/ascend_ex/__init__.py b/test/connector/exchange/ascend_ex/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/connector/exchange/bitmax/test_bitmax_auth.py b/test/connector/exchange/ascend_ex/test_ascend_ex_auth.py similarity index 72% rename from test/connector/exchange/bitmax/test_bitmax_auth.py rename to test/connector/exchange/ascend_ex/test_ascend_ex_auth.py index 46a68bfae7..59d866ad87 100644 --- a/test/connector/exchange/bitmax/test_bitmax_auth.py +++ b/test/connector/exchange/ascend_ex/test_ascend_ex_auth.py @@ -9,21 +9,22 @@ import logging from os.path import join, realpath from typing import Dict, Any -from hummingbot.connector.exchange.bitmax.bitmax_auth import BitmaxAuth +from hummingbot.connector.exchange.ascend_ex.ascend_ex_auth import AscendExAuth from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL -from hummingbot.connector.exchange.bitmax.bitmax_constants import REST_URL, getWsUrlPriv +from hummingbot.connector.exchange.ascend_ex.ascend_ex_constants import REST_URL +from hummingbot.connector.exchange.ascend_ex.ascend_ex_util import get_rest_url_private sys.path.insert(0, realpath(join(__file__, "../../../../../"))) logging.basicConfig(level=METRICS_LOG_LEVEL) -class TestAuth(unittest.TestCase): +class TestAscendExAuth(unittest.TestCase): @classmethod def setUpClass(cls): cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() - api_key = conf.bitmax_api_key - secret_key = conf.bitmax_secret_key - cls.auth = BitmaxAuth(api_key, secret_key) + api_key = conf.ascend_ex_api_key + secret_key = conf.ascend_ex_secret_key + cls.auth = AscendExAuth(api_key, secret_key) async def rest_auth(self) -> Dict[Any, Any]: headers = { @@ -37,7 +38,7 @@ async def ws_auth(self) -> Dict[Any, Any]: info = await self.rest_auth() accountGroup = info.get("data").get("accountGroup") headers = self.auth.get_auth_headers("stream") - ws = await websockets.connect(f"{getWsUrlPriv(accountGroup)}/stream", extra_headers=headers) + ws = await websockets.connect(f"{get_rest_url_private(accountGroup)}/stream", extra_headers=headers) raw_msg = await asyncio.wait_for(ws.recv(), 5000) msg = ujson.loads(raw_msg) diff --git a/test/connector/exchange/bitmax/test_bitmax_exchange.py b/test/connector/exchange/ascend_ex/test_ascend_ex_exchange.py similarity index 97% rename from test/connector/exchange/bitmax/test_bitmax_exchange.py rename to test/connector/exchange/ascend_ex/test_ascend_ex_exchange.py index ca642fe385..e3c928520d 100644 --- a/test/connector/exchange/bitmax/test_bitmax_exchange.py +++ b/test/connector/exchange/ascend_ex/test_ascend_ex_exchange.py @@ -1,15 +1,17 @@ from os.path import join, realpath import sys; sys.path.insert(0, realpath(join(__file__, "../../../../../"))) + import asyncio -import logging -from decimal import Decimal -import unittest +import conf import contextlib -import time +import logging +import math import os +import time +import unittest + +from decimal import Decimal from typing import List -import conf -import math from hummingbot.core.clock import Clock, ClockMode from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL @@ -33,15 +35,15 @@ from hummingbot.model.order import Order from hummingbot.model.trade_fill import TradeFill from hummingbot.connector.markets_recorder import MarketsRecorder -from hummingbot.connector.exchange.bitmax.bitmax_exchange import BitmaxExchange +from hummingbot.connector.exchange.ascend_ex.ascend_ex_exchange import AscendExExchange logging.basicConfig(level=METRICS_LOG_LEVEL) -API_KEY = conf.bitmax_api_key -API_SECRET = conf.bitmax_secret_key +API_KEY = conf.ascend_ex_api_key +API_SECRET = conf.ascend_ex_secret_key -class BitmaxExchangeUnitTest(unittest.TestCase): +class AscendExExchangeUnitTest(unittest.TestCase): events: List[MarketEvent] = [ MarketEvent.BuyOrderCompleted, MarketEvent.SellOrderCompleted, @@ -52,7 +54,7 @@ class BitmaxExchangeUnitTest(unittest.TestCase): MarketEvent.OrderCancelled, MarketEvent.OrderFailure ] - connector: BitmaxExchange + connector: AscendExExchange event_logger: EventLogger trading_pair = "BTC-USDT" base_token, quote_token = trading_pair.split("-") @@ -65,13 +67,13 @@ def setUpClass(cls): cls.ev_loop = asyncio.get_event_loop() cls.clock: Clock = Clock(ClockMode.REALTIME) - cls.connector: BitmaxExchange = BitmaxExchange( - bitmax_api_key=API_KEY, - bitmax_secret_key=API_SECRET, + cls.connector: AscendExExchange = AscendExExchange( + ascend_ex_api_key=API_KEY, + ascend_ex_secret_key=API_SECRET, trading_pairs=[cls.trading_pair], trading_required=True ) - print("Initializing Bitmax market... this will take about a minute.") + print("Initializing AscendEx exchange... this will take about a minute.") cls.clock.add_iterator(cls.connector) cls.stack: contextlib.ExitStack = contextlib.ExitStack() cls._clock = cls.stack.enter_context(cls.clock) @@ -336,7 +338,7 @@ def test_orders_saving_and_restoration(self): self.clock.remove_iterator(self.connector) for event_tag in self.events: self.connector.remove_listener(event_tag, self.event_logger) - new_connector = BitmaxExchange(API_KEY, API_SECRET, [self.trading_pair], True) + new_connector = AscendExExchange(API_KEY, API_SECRET, [self.trading_pair], True) for event_tag in self.events: new_connector.add_listener(event_tag, self.event_logger) recorder.stop() diff --git a/test/connector/exchange/bitmax/test_bitmax_order_book_tracker.py b/test/connector/exchange/ascend_ex/test_ascend_ex_order_book_tracker.py similarity index 88% rename from test/connector/exchange/bitmax/test_bitmax_order_book_tracker.py rename to test/connector/exchange/ascend_ex/test_ascend_ex_order_book_tracker.py index 331860fbba..ed19775c0d 100755 --- a/test/connector/exchange/bitmax/test_bitmax_order_book_tracker.py +++ b/test/connector/exchange/ascend_ex/test_ascend_ex_order_book_tracker.py @@ -9,8 +9,8 @@ from typing import Dict, Optional, List from hummingbot.core.event.event_logger import EventLogger from hummingbot.core.event.events import OrderBookEvent, OrderBookTradeEvent, TradeType -from hummingbot.connector.exchange.bitmax.bitmax_order_book_tracker import BitmaxOrderBookTracker -from hummingbot.connector.exchange.bitmax.bitmax_api_order_book_data_source import BitmaxAPIOrderBookDataSource +from hummingbot.connector.exchange.ascend_ex.ascend_ex_order_book_tracker import AscendExOrderBookTracker +from hummingbot.connector.exchange.ascend_ex.ascend_ex_api_order_book_data_source import AscendExAPIOrderBookDataSource from hummingbot.core.data_type.order_book import OrderBook from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL @@ -19,8 +19,8 @@ logging.basicConfig(level=METRICS_LOG_LEVEL) -class BitmaxOrderBookTrackerUnitTest(unittest.TestCase): - order_book_tracker: Optional[BitmaxOrderBookTracker] = None +class AscendExOrderBookTrackerUnitTest(unittest.TestCase): + order_book_tracker: Optional[AscendExOrderBookTracker] = None events: List[OrderBookEvent] = [ OrderBookEvent.TradeEvent ] @@ -32,7 +32,7 @@ class BitmaxOrderBookTrackerUnitTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() - cls.order_book_tracker: BitmaxOrderBookTracker = BitmaxOrderBookTracker(cls.trading_pairs) + cls.order_book_tracker: AscendExOrderBookTracker = AscendExOrderBookTracker(cls.trading_pairs) cls.order_book_tracker.start() cls.ev_loop.run_until_complete(cls.wait_til_tracker_ready()) @@ -96,7 +96,7 @@ def test_tracker_integrity(self): def test_api_get_last_traded_prices(self): prices = self.ev_loop.run_until_complete( - BitmaxAPIOrderBookDataSource.get_last_traded_prices(["BTC-USDT", "LTC-BTC"])) + AscendExAPIOrderBookDataSource.get_last_traded_prices(["BTC-USDT", "LTC-BTC"])) for key, value in prices.items(): print(f"{key} last_trade_price: {value}") self.assertGreater(prices["BTC-USDT"], 1000) diff --git a/test/connector/exchange/bitmax/test_bitmax_user_stream_tracker.py b/test/connector/exchange/ascend_ex/test_ascend_ex_user_stream_tracker.py similarity index 58% rename from test/connector/exchange/bitmax/test_bitmax_user_stream_tracker.py rename to test/connector/exchange/ascend_ex/test_ascend_ex_user_stream_tracker.py index 1b4cb5c84b..ce107d09f2 100644 --- a/test/connector/exchange/bitmax/test_bitmax_user_stream_tracker.py +++ b/test/connector/exchange/ascend_ex/test_ascend_ex_user_stream_tracker.py @@ -6,8 +6,8 @@ import conf from os.path import join, realpath -from hummingbot.connector.exchange.bitmax.bitmax_user_stream_tracker import BitmaxUserStreamTracker -from hummingbot.connector.exchange.bitmax.bitmax_auth import BitmaxAuth +from hummingbot.connector.exchange.ascend_ex.ascend_ex_user_stream_tracker import AscendExUserStreamTracker +from hummingbot.connector.exchange.ascend_ex.ascend_ex_auth import AscendExAuth from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL @@ -16,17 +16,17 @@ logging.basicConfig(level=METRICS_LOG_LEVEL) -class BitmaxUserStreamTrackerUnitTest(unittest.TestCase): - api_key = conf.bitmax_api_key - api_secret = conf.bitmax_secret_key +class AscendExUserStreamTrackerUnitTest(unittest.TestCase): + api_key = conf.ascend_ex_api_key + api_secret = conf.ascend_ex_secret_key @classmethod def setUpClass(cls): cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() - cls.bitmax_auth = BitmaxAuth(cls.api_key, cls.api_secret) + cls.ascend_ex_auth = AscendExAuth(cls.api_key, cls.api_secret) cls.trading_pairs = ["BTC-USDT"] - cls.user_stream_tracker: BitmaxUserStreamTracker = BitmaxUserStreamTracker( - bitmax_auth=cls.bitmax_auth, trading_pairs=cls.trading_pairs) + cls.user_stream_tracker: AscendExUserStreamTracker = AscendExUserStreamTracker( + ascend_ex_auth=cls.ascend_ex_auth, trading_pairs=cls.trading_pairs) cls.user_stream_tracker_task: asyncio.Task = safe_ensure_future(cls.user_stream_tracker.start()) def test_user_stream(self): diff --git a/test/connector/exchange/digifinex/__init__.py b/test/connector/exchange/digifinex/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/connector/exchange/digifinex/fixture.py b/test/connector/exchange/digifinex/fixture.py new file mode 100644 index 0000000000..85390bb01f --- /dev/null +++ b/test/connector/exchange/digifinex/fixture.py @@ -0,0 +1,127 @@ +BALANCES = { + 'id': 815129178419638016, 'method': 'private/get-account-summary', 'code': 0, + 'result': { + 'accounts': [{'balance': 50, 'available': 50, 'order': 0.0, 'stake': 0, 'currency': 'USDT'}, + {'balance': 0.002, 'available': 0.002, 'order': 0, 'stake': 0, 'currency': 'BTC'}] + } +} + +INSTRUMENTS = {'id': -1, 'method': 'public/get-instruments', 'code': 0, 'result': {'instruments': [ + {'instrument_name': 'BTC_USDT', 'quote_currency': 'USDT', 'base_currency': 'BTC', 'price_decimals': 2, + 'quantity_decimals': 6}, + {'instrument_name': 'ETH_USDT', 'quote_currency': 'USDT', 'base_currency': 'ETH', 'price_decimals': 2, + 'quantity_decimals': 5}, +]}} + +TICKERS = {'code': 0, 'method': 'public/get-ticker', 'result': { + 'instrument_name': 'BTC_USDT', + 'data': [{'i': 'BTC_USDT', 'b': 11490.0, 'k': 11492.05, + 'a': 11490.0, 't': 1598674849297, + 'v': 754.531926, 'h': 11546.11, 'l': 11366.62, + 'c': 104.19}]}} + +GET_BOOK = { + "code": 0, "method": "public/get-book", "result": { + "instrument_name": "BTC_USDT", "depth": 5, "data": + [{"bids": [[11490.00, 0.010676, 1], [11488.34, 0.055374, 1], [11487.47, 0.003000, 1], + [11486.50, 0.031032, 1], + [11485.97, 0.087074, 1]], + "asks": [[11492.05, 0.232044, 1], [11492.06, 0.497900, 1], [11493.12, 2.005693, 1], + [11494.12, 7.000000, 1], + [11494.41, 0.032853, 1]], "t": 1598676097390}]}} + +PLACE_ORDER = {'id': 632194937848317440, 'method': 'private/create-order', 'code': 0, + 'result': {'order_id': '1', 'client_oid': 'buy-BTC-USDT-1598607082008742'}} + +CANCEL = {'id': 31484728768575776, 'method': 'private/cancel-order', 'code': 0} + +UNFILLED_ORDER = { + 'id': 798015906490506624, + 'method': 'private/get-order-detail', + 'code': 0, + 'result': { + 'trade_list': [], + 'order_info': { + 'status': 'ACTIVE', + 'side': 'BUY', + 'price': 9164.82, + 'quantity': 0.0001, + 'order_id': '1', + 'client_oid': 'buy-BTC-USDT-1598607082008742', + 'create_time': 1598607082329, + 'update_time': 1598607082332, + 'type': 'LIMIT', + 'instrument_name': 'BTC_USDT', + 'avg_price': 0.0, + 'cumulative_quantity': 0.0, + 'cumulative_value': 0.0, + 'fee_currency': 'BTC', + 'exec_inst': 'POST_ONLY', + 'time_in_force': 'GOOD_TILL_CANCEL'} + } +} + +WS_INITIATED = {'id': 317343764453238848, 'method': 'public/auth', 'code': 0} +WS_SUBSCRIBE = {'id': 802984382214439040, 'method': 'subscribe', 'code': 0} +WS_HEARTBEAT = {'id': 1598755526207, 'method': 'public/heartbeat'} + +WS_ORDER_FILLED = { + 'id': -1, 'method': 'subscribe', 'code': 0, + 'result': { + 'instrument_name': 'BTC_USDT', + 'subscription': 'user.order.BTC_USDT', + 'channel': 'user.order', + 'data': [ + {'status': 'FILLED', + 'side': 'BUY', + 'price': 12080.9, + 'quantity': 0.0001, + 'order_id': '1', + 'client_oid': 'buy-BTC-USDT-1598681216010994', + 'create_time': 1598681216332, + 'update_time': 1598681216334, + 'type': 'LIMIT', + 'instrument_name': 'BTC_USDT', + 'avg_price': 11505.62, + 'cumulative_quantity': 0.0001, + 'cumulative_value': 11.50562, + 'fee_currency': 'BTC', + 'exec_inst': '', + 'time_in_force': 'GOOD_TILL_CANCEL'}]}} + +WS_TRADE = { + 'id': -1, 'method': 'subscribe', 'code': 0, + 'result': { + 'instrument_name': 'BTC_USDT', + 'subscription': 'user.trade.BTC_USDT', + 'channel': 'user.trade', + 'data': [ + {'side': 'BUY', + 'fee': 1.6e-06, + 'trade_id': '699422550491763776', + 'instrument_name': 'BTC_USDT', + 'create_time': 1598681216334, + 'traded_price': 11505.62, + 'traded_quantity': 0.0001, + 'fee_currency': 'BTC', + 'order_id': '1'}]}} + +WS_BALANCE = { + 'id': -1, 'method': 'subscribe', 'code': 0, + 'result': { + 'subscription': 'user.balance', 'channel': 'user.balance', + 'data': [{'balance': 47, 'available': 46, + 'order': 1, 'stake': 0, + 'currency': 'USDT'}]}} + +WS_ORDER_CANCELLED = { + 'id': -1, 'method': 'subscribe', 'code': 0, + 'result': { + 'instrument_name': 'BTC_USDT', 'subscription': 'user.order.BTC_USDT', + 'channel': 'user.order', 'data': [ + {'status': 'CANCELED', 'side': 'BUY', 'price': 13918.12, 'quantity': 0.0001, + 'order_id': '1', 'client_oid': 'buy-BTC-USDT-1598757896008300', + 'create_time': 1598757896312, 'update_time': 1598757896312, 'type': 'LIMIT', + 'instrument_name': 'BTC_USDT', 'avg_price': 0.0, 'cumulative_quantity': 0.0, + 'cumulative_value': 0.0, 'fee_currency': 'BTC', 'exec_inst': 'POST_ONLY', + 'time_in_force': 'GOOD_TILL_CANCEL'}]}} diff --git a/test/connector/exchange/digifinex/test_digifinex_auth.py b/test/connector/exchange/digifinex/test_digifinex_auth.py new file mode 100644 index 0000000000..4fcbc59c13 --- /dev/null +++ b/test/connector/exchange/digifinex/test_digifinex_auth.py @@ -0,0 +1,32 @@ +from os.path import join, realpath +import sys; sys.path.insert(0, realpath(join(__file__, "../../../../../"))) + +import asyncio +import unittest + +import conf +from hummingbot.connector.exchange.digifinex.digifinex_auth import DigifinexAuth +from hummingbot.connector.exchange.digifinex.digifinex_websocket import DigifinexWebsocket + + +class TestAuth(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + api_key = conf.digifinex_api_key + secret_key = conf.digifinex_secret_key + cls.auth = DigifinexAuth(api_key, secret_key) + cls.ws = DigifinexWebsocket(cls.auth) + + async def ws_auth(self): + await self.ws.connect() + await self.ws.subscribe("balance", ["USDT", "BTC", "ETH"]) + + # no msg will arrive until balance changed after subscription + # async for response in self.ws.on_message(): + # if (response.get("method") == "subscribe"): + # return response + + def test_ws_auth(self): + self.ev_loop.run_until_complete(self.ws_auth()) + # assert result["code"] == 0 diff --git a/test/connector/exchange/digifinex/test_digifinex_exchange.py b/test/connector/exchange/digifinex/test_digifinex_exchange.py new file mode 100644 index 0000000000..54f9be5c57 --- /dev/null +++ b/test/connector/exchange/digifinex/test_digifinex_exchange.py @@ -0,0 +1,539 @@ +# print('__file__={0:<35} | __name__={1:<20} | __package__={2:<20}'.format(__file__,__name__,str(__package__))) +import os +from os.path import join, realpath +import sys; sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +import asyncio +import logging +from decimal import Decimal +import unittest +import contextlib +import time +from typing import List +# from unittest import mock +import conf +import math + +from test.connector.exchange.digifinex import fixture +from hummingbot.core.clock import Clock, ClockMode +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL +from hummingbot.core.utils.async_utils import safe_gather, safe_ensure_future +from hummingbot.core.event.event_logger import EventLogger +from hummingbot.core.event.events import ( + BuyOrderCompletedEvent, + BuyOrderCreatedEvent, + MarketEvent, + OrderFilledEvent, + OrderType, + SellOrderCompletedEvent, + SellOrderCreatedEvent, + OrderCancelledEvent +) +from hummingbot.model.sql_connection_manager import ( + SQLConnectionManager, + SQLConnectionType +) +from hummingbot.model.market_state import MarketState +from hummingbot.model.order import Order +from hummingbot.model.trade_fill import TradeFill +from hummingbot.connector.markets_recorder import MarketsRecorder +from hummingbot.connector.exchange.digifinex.digifinex_exchange import DigifinexExchange +# from hummingbot.connector.exchange.digifinex.digifinex_constants import WSS_PUBLIC_URL, WSS_PRIVATE_URL +# from test.integration.humming_web_app import HummingWebApp +# from test.integration.humming_ws_server import HummingWsServerFactory + +# API_MOCK_ENABLED = conf.mock_api_enabled is not None and conf.mock_api_enabled.lower() in ['true', 'yes', '1'] +API_MOCK_ENABLED = False + +logging.basicConfig(level=METRICS_LOG_LEVEL) +# logging.basicConfig(level=logging.NETWORK) +# logging.basicConfig(level=logging.DEBUG) +API_KEY = conf.digifinex_api_key +API_SECRET = conf.digifinex_secret_key +# BASE_API_URL = "openapi.digifinex.com" + + +class DigifinexExchangeUnitTest(unittest.TestCase): + events: List[MarketEvent] = [ + MarketEvent.BuyOrderCompleted, + MarketEvent.SellOrderCompleted, + MarketEvent.OrderFilled, + MarketEvent.TransactionFailure, + MarketEvent.BuyOrderCreated, + MarketEvent.SellOrderCreated, + MarketEvent.OrderCancelled, + MarketEvent.OrderFailure + ] + connector: DigifinexExchange + event_logger: EventLogger + trading_pair = "BTC-USDT" + base_token, quote_token = trading_pair.split("-") + stack: contextlib.ExitStack + sql: SQLConnectionManager + + @classmethod + def setUpClass(cls): + global MAINNET_RPC_URL + + cls.ev_loop = asyncio.get_event_loop() + + if API_MOCK_ENABLED: + raise NotImplementedError() + # cls.web_app = HummingWebApp.get_instance() + # cls.web_app.add_host_to_mock(BASE_API_URL, []) + # cls.web_app.start() + # cls.ev_loop.run_until_complete(cls.web_app.wait_til_started()) + # cls._patcher = mock.patch("aiohttp.client.URL") + # cls._url_mock = cls._patcher.start() + # cls._url_mock.side_effect = cls.web_app.reroute_local + # cls.web_app.update_response("get", BASE_API_URL, "/v2/public/get-ticker", fixture.TICKERS) + # cls.web_app.update_response("get", BASE_API_URL, "/v2/public/get-instruments", fixture.INSTRUMENTS) + # cls.web_app.update_response("get", BASE_API_URL, "/v2/public/get-book", fixture.GET_BOOK) + # cls.web_app.update_response("post", BASE_API_URL, "/v2/private/get-account-summary", fixture.BALANCES) + # cls.web_app.update_response("post", BASE_API_URL, "/v2/private/cancel-order", fixture.CANCEL) + + # HummingWsServerFactory.start_new_server(WSS_PRIVATE_URL) + # HummingWsServerFactory.start_new_server(WSS_PUBLIC_URL) + # cls._ws_patcher = unittest.mock.patch("websockets.connect", autospec=True) + # cls._ws_mock = cls._ws_patcher.start() + # cls._ws_mock.side_effect = HummingWsServerFactory.reroute_ws_connect + + cls.clock: Clock = Clock(ClockMode.REALTIME) + cls.connector: DigifinexExchange = DigifinexExchange( + digifinex_api_key=API_KEY, + digifinex_secret_key=API_SECRET, + trading_pairs=[cls.trading_pair], + trading_required=True + ) + print("Initializing Digifinex market... this will take about a minute.") + cls.clock.add_iterator(cls.connector) + cls.stack: contextlib.ExitStack = contextlib.ExitStack() + cls._clock = cls.stack.enter_context(cls.clock) + # if API_MOCK_ENABLED: + # HummingWsServerFactory.send_json_threadsafe(WSS_PRIVATE_URL, fixture.WS_INITIATED, delay=0.5) + # HummingWsServerFactory.send_json_threadsafe(WSS_PRIVATE_URL, fixture.WS_SUBSCRIBE, delay=0.51) + # HummingWsServerFactory.send_json_threadsafe(WSS_PRIVATE_URL, fixture.WS_HEARTBEAT, delay=0.52) + + cls.ev_loop.run_until_complete(cls.wait_til_ready()) + print("Ready.") + + @classmethod + def tearDownClass(cls) -> None: + cls.stack.close() + # if API_MOCK_ENABLED: + # cls.web_app.stop() + # cls._patcher.stop() + # cls._ws_patcher.stop() + + @classmethod + async def wait_til_ready(cls, connector = None): + if connector is None: + connector = cls.connector + while True: + now = time.time() + next_iteration = now // 1.0 + 1 + if connector.ready: + break + else: + await cls._clock.run_til(next_iteration) + await asyncio.sleep(1.0) + + def setUp(self): + self.db_path: str = realpath(join(__file__, "../connector_test.sqlite")) + try: + # on windows cannot unlink the sqlite db file before closing the db + if os.name != 'nt': + os.unlink(self.db_path) + except FileNotFoundError: + pass + + self.event_logger = EventLogger() + for event_tag in self.events: + self.connector.add_listener(event_tag, self.event_logger) + + def tearDown(self): + for event_tag in self.events: + self.connector.remove_listener(event_tag, self.event_logger) + self.event_logger = None + # self.sql._engine.dispose() + + async def run_parallel_async(self, *tasks): + future: asyncio.Future = safe_ensure_future(safe_gather(*tasks)) + while not future.done(): + now = time.time() + next_iteration = now // 1.0 + 1 + await self._clock.run_til(next_iteration) + await asyncio.sleep(1.0) + return future.result() + + def run_parallel(self, *tasks): + return self.ev_loop.run_until_complete(self.run_parallel_async(*tasks)) + + def test_estimate_fee(self): + maker_fee = self.connector.estimate_fee_pct(True) + self.assertAlmostEqual(maker_fee, Decimal("0.001")) + taker_fee = self.connector.estimate_fee_pct(False) + self.assertAlmostEqual(taker_fee, Decimal("0.001")) + + def _place_order(self, is_buy, amount, order_type, price, ex_order_id, get_order_fixture=None, + ws_trade_fixture=None, ws_order_fixture=None) -> str: + # if API_MOCK_ENABLED: + # data = fixture.PLACE_ORDER.copy() + # data["result"]["order_id"] = str(ex_order_id) + # self.web_app.update_response("post", BASE_API_URL, "/v2/private/create-order", data) + if is_buy: + cl_order_id = self.connector.buy(self.trading_pair, amount, order_type, price) + else: + cl_order_id = self.connector.sell(self.trading_pair, amount, order_type, price) + # if API_MOCK_ENABLED: + # if get_order_fixture is not None: + # data = get_order_fixture.copy() + # data["result"]["order_info"]["client_oid"] = cl_order_id + # data["result"]["order_info"]["order_id"] = ex_order_id + # self.web_app.update_response("post", BASE_API_URL, "/v2/private/get-order-detail", data) + # if ws_trade_fixture is not None: + # data = ws_trade_fixture.copy() + # data["result"]["data"][0]["order_id"] = str(ex_order_id) + # HummingWsServerFactory.send_json_threadsafe(WSS_PRIVATE_URL, data, delay=0.1) + # if ws_order_fixture is not None: + # data = ws_order_fixture.copy() + # data["result"]["data"][0]["order_id"] = str(ex_order_id) + # data["result"]["data"][0]["client_oid"] = cl_order_id + # HummingWsServerFactory.send_json_threadsafe(WSS_PRIVATE_URL, data, delay=0.12) + return cl_order_id + + def _cancel_order(self, cl_order_id): + self.connector.cancel(self.trading_pair, cl_order_id) + # if API_MOCK_ENABLED: + # data = fixture.WS_ORDER_CANCELLED.copy() + # data["result"]["data"][0]["client_oid"] = cl_order_id + # HummingWsServerFactory.send_json_threadsafe(WSS_PRIVATE_URL, data, delay=0.1) + + def test_buy_and_sell(self): + self.ev_loop.run_until_complete(self.connector.cancel_all(0)) + + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.05") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + quote_bal = self.connector.get_available_balance(self.quote_token) + base_bal = self.connector.get_available_balance(self.base_token) + + order_id = self._place_order(True, amount, OrderType.LIMIT, price, 1, None, + fixture.WS_TRADE) + order_completed_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(2)) + trade_events = [t for t in self.event_logger.event_log if isinstance(t, OrderFilledEvent)] + base_amount_traded = sum(t.amount for t in trade_events) + quote_amount_traded = sum(t.amount * t.price for t in trade_events) + + self.assertTrue([evt.order_type == OrderType.LIMIT for evt in trade_events]) + self.assertEqual(order_id, order_completed_event.order_id) + self.assertEqual(amount, order_completed_event.base_asset_amount) + self.assertEqual("BTC", order_completed_event.base_asset) + self.assertEqual("USDT", order_completed_event.quote_asset) + self.assertAlmostEqual(base_amount_traded, order_completed_event.base_asset_amount) + self.assertAlmostEqual(quote_amount_traded, order_completed_event.quote_asset_amount) + # todo: get fee + # self.assertGreater(order_completed_event.fee_amount, Decimal(0)) + self.assertTrue(any([isinstance(event, BuyOrderCreatedEvent) and event.order_id == order_id + for event in self.event_logger.event_log])) + + # check available quote balance gets updated, we need to wait a bit for the balance message to arrive + expected_quote_bal = quote_bal - quote_amount_traded + # self._mock_ws_bal_update(self.quote_token, expected_quote_bal) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.assertAlmostEqual(expected_quote_bal, self.connector.get_available_balance(self.quote_token), delta=0.1) + + # Reset the logs + self.event_logger.clear() + + # Try to sell back the same amount to the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.95") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + order_id = self._place_order(False, amount, OrderType.LIMIT, price, 2, None, + fixture.WS_TRADE) + order_completed_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCompletedEvent)) + trade_events = [t for t in self.event_logger.event_log if isinstance(t, OrderFilledEvent)] + base_amount_traded = sum(t.amount for t in trade_events) + quote_amount_traded = sum(t.amount * t.price for t in trade_events) + + self.assertTrue([evt.order_type == OrderType.LIMIT for evt in trade_events]) + self.assertEqual(order_id, order_completed_event.order_id) + self.assertEqual(amount, order_completed_event.base_asset_amount) + self.assertEqual("BTC", order_completed_event.base_asset) + self.assertEqual("USDT", order_completed_event.quote_asset) + self.assertAlmostEqual(base_amount_traded, order_completed_event.base_asset_amount) + self.assertAlmostEqual(quote_amount_traded, order_completed_event.quote_asset_amount) + # todo: get fee + # self.assertGreater(order_completed_event.fee_amount, Decimal(0)) + self.assertTrue(any([isinstance(event, SellOrderCreatedEvent) and event.order_id == order_id + for event in self.event_logger.event_log])) + + # check available base balance gets updated, we need to wait a bit for the balance message to arrive + expected_base_bal = base_bal + # self._mock_ws_bal_update(self.base_token, expected_base_bal) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.assertAlmostEqual(expected_base_bal, self.connector.get_available_balance(self.base_token), 5) + + def test_limit_makers_unfilled(self): + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.8") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.00005")) + quote_bal = self.connector.get_available_balance(self.quote_token) + + # order_id = self.connector.buy(self.trading_pair, amount, OrderType.LIMIT_MAKER, price) + cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1, fixture.UNFILLED_ORDER) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + # check available quote balance gets updated, we need to wait a bit for the balance message to arrive + expected_quote_bal = quote_bal - (price * amount) + # self._mock_ws_bal_update(self.quote_token, expected_quote_bal) + self.ev_loop.run_until_complete(asyncio.sleep(2)) + self.assertAlmostEqual(expected_quote_bal, self.connector.get_available_balance(self.quote_token), 1) + self._cancel_order(cl_order_id) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.2") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + cl_order_id = self._place_order(False, amount, OrderType.LIMIT_MAKER, price, 2, fixture.UNFILLED_ORDER) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + self._cancel_order(cl_order_id) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + # def _mock_ws_bal_update(self, token, available): + # if API_MOCK_ENABLED: + # available = float(available) + # data = fixture.WS_BALANCE.copy() + # data["result"]["data"][0]["currency"] = token + # data["result"]["data"][0]["available"] = available + # HummingWsServerFactory.send_json_threadsafe(WSS_PRIVATE_URL, fixture.WS_BALANCE, delay=0.1) + + def test_limit_maker_rejections(self): + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.2") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1, None, None, + fixture.WS_ORDER_CANCELLED) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + price = self.connector.get_price(self.trading_pair, False) * Decimal("0.8") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + cl_order_id = self._place_order(False, amount, OrderType.LIMIT_MAKER, price, 2, None, None, + fixture.WS_ORDER_CANCELLED) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + def test_cancel_all(self): + bid_price = self.connector.get_price(self.trading_pair, True) + ask_price = self.connector.get_price(self.trading_pair, False) + bid_price = self.connector.quantize_order_price(self.trading_pair, bid_price * Decimal("0.7")) + ask_price = self.connector.quantize_order_price(self.trading_pair, ask_price * Decimal("1.5")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + buy_id = self._place_order(True, amount, OrderType.LIMIT, bid_price, 1) + sell_id = self._place_order(False, amount, OrderType.LIMIT, ask_price, 2) + + self.ev_loop.run_until_complete(asyncio.sleep(1)) + asyncio.ensure_future(self.connector.cancel_all(3)) + # if API_MOCK_ENABLED: + # data = fixture.WS_ORDER_CANCELLED.copy() + # data["result"]["data"][0]["client_oid"] = buy_id + # data["result"]["data"][0]["order_id"] = 1 + # HummingWsServerFactory.send_json_threadsafe(WSS_PRIVATE_URL, data, delay=0.1) + # self.ev_loop.run_until_complete(asyncio.sleep(1)) + # data = fixture.WS_ORDER_CANCELLED.copy() + # data["result"]["data"][0]["client_oid"] = sell_id + # data["result"]["data"][0]["order_id"] = 2 + # HummingWsServerFactory.send_json_threadsafe(WSS_PRIVATE_URL, data, delay=0.11) + self.ev_loop.run_until_complete(asyncio.sleep(3)) + cancel_events = [t for t in self.event_logger.event_log if isinstance(t, OrderCancelledEvent)] + self.assertEqual({buy_id, sell_id}, {o.order_id for o in cancel_events}) + + def test_order_price_precision(self): + bid_price: Decimal = self.connector.get_price(self.trading_pair, True) + ask_price: Decimal = self.connector.get_price(self.trading_pair, False) + mid_price: Decimal = (bid_price + ask_price) / 2 + amount: Decimal = Decimal("0.000123456") + + # Make sure there's enough balance to make the limit orders. + self.assertGreater(self.connector.get_balance("BTC"), Decimal("0.001")) + self.assertGreater(self.connector.get_balance("USDT"), Decimal("10")) + + # Intentionally set some prices with too many decimal places s.t. they + # need to be quantized. Also, place them far away from the mid-price s.t. they won't + # get filled during the test. + bid_price = mid_price * Decimal("0.9333192292111341") + ask_price = mid_price * Decimal("1.0492431474884933") + + cl_order_id_1 = self._place_order(True, amount, OrderType.LIMIT, bid_price, 1, fixture.UNFILLED_ORDER) + + # Wait for the order created event and examine the order made + self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + order = self.connector.in_flight_orders[cl_order_id_1] + quantized_bid_price = self.connector.quantize_order_price(self.trading_pair, bid_price) + quantized_bid_size = self.connector.quantize_order_amount(self.trading_pair, amount) + self.assertEqual(quantized_bid_price, order.price) + self.assertEqual(quantized_bid_size, order.amount) + + # Test ask order + cl_order_id_2 = self._place_order(False, amount, OrderType.LIMIT, ask_price, 1, fixture.UNFILLED_ORDER) + + # Wait for the order created event and examine and order made + self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCreatedEvent)) + order = self.connector.in_flight_orders[cl_order_id_2] + quantized_ask_price = self.connector.quantize_order_price(self.trading_pair, Decimal(ask_price)) + quantized_ask_size = self.connector.quantize_order_amount(self.trading_pair, Decimal(amount)) + self.assertEqual(quantized_ask_price, order.price) + self.assertEqual(quantized_ask_size, order.amount) + + self._cancel_order(cl_order_id_1) + self._cancel_order(cl_order_id_2) + + def test_orders_saving_and_restoration(self): + config_path = "test_config" + strategy_name = "test_strategy" + sql = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, db_path=self.db_path) + order_id = None + recorder = MarketsRecorder(sql, [self.connector], config_path, strategy_name) + recorder.start() + + try: + self.connector._in_flight_orders.clear() + self.assertEqual(0, len(self.connector.tracking_states)) + + # Try to put limit buy order for 0.02 ETH worth of ZRX, and watch for order creation event. + current_bid_price: Decimal = self.connector.get_price(self.trading_pair, True) + price: Decimal = current_bid_price * Decimal("0.8") + price = self.connector.quantize_order_price(self.trading_pair, price) + + amount: Decimal = Decimal("0.0001") + amount = self.connector.quantize_order_amount(self.trading_pair, amount) + + cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1, fixture.UNFILLED_ORDER) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + + # Verify tracking states + self.assertEqual(1, len(self.connector.tracking_states)) + self.assertEqual(cl_order_id, list(self.connector.tracking_states.keys())[0]) + + # Verify orders from recorder + recorded_orders: List[Order] = recorder.get_orders_for_config_and_market(config_path, self.connector) + self.assertEqual(1, len(recorded_orders)) + self.assertEqual(cl_order_id, recorded_orders[0].id) + + # Verify saved market states + saved_market_states: MarketState = recorder.get_market_states(config_path, self.connector) + self.assertIsNotNone(saved_market_states) + self.assertIsInstance(saved_market_states.saved_state, dict) + self.assertGreater(len(saved_market_states.saved_state), 0) + + # Close out the current market and start another market. + self.connector.stop(self._clock) + self.ev_loop.run_until_complete(asyncio.sleep(5)) + self.clock.remove_iterator(self.connector) + for event_tag in self.events: + self.connector.remove_listener(event_tag, self.event_logger) + new_connector = DigifinexExchange(API_KEY, API_SECRET, [self.trading_pair], True) + for event_tag in self.events: + new_connector.add_listener(event_tag, self.event_logger) + recorder.stop() + recorder = MarketsRecorder(sql, [new_connector], config_path, strategy_name) + recorder.start() + saved_market_states = recorder.get_market_states(config_path, new_connector) + self.clock.add_iterator(new_connector) + if not API_MOCK_ENABLED: + self.ev_loop.run_until_complete(self.wait_til_ready(new_connector)) + self.assertEqual(0, len(new_connector.limit_orders)) + self.assertEqual(0, len(new_connector.tracking_states)) + new_connector.restore_tracking_states(saved_market_states.saved_state) + self.assertEqual(1, len(new_connector.limit_orders)) + self.assertEqual(1, len(new_connector.tracking_states)) + + # Cancel the order and verify that the change is saved. + self._cancel_order(cl_order_id) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + order_id = None + self.assertEqual(0, len(new_connector.limit_orders)) + self.assertEqual(0, len(new_connector.tracking_states)) + saved_market_states = recorder.get_market_states(config_path, new_connector) + self.assertEqual(0, len(saved_market_states.saved_state)) + finally: + if order_id is not None: + self.connector.cancel(self.trading_pair, cl_order_id) + self.run_parallel(self.event_logger.wait_for(OrderCancelledEvent)) + + recorder.stop() + # sql._engine.dispose() + # on windows cannot unlink the sqlite db file before closing the db + if os.name != 'nt': + os.unlink(self.db_path) + + def test_update_last_prices(self): + # This is basic test to see if order_book last_trade_price is initiated and updated. + for order_book in self.connector.order_books.values(): + for _ in range(5): + self.ev_loop.run_until_complete(asyncio.sleep(1)) + print(order_book.last_trade_price) + self.assertFalse(math.isnan(order_book.last_trade_price)) + + def test_filled_orders_recorded(self): + config_path: str = "test_config" + strategy_name: str = "test_strategy" + sql = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, db_path=self.db_path) + order_id = None + recorder = MarketsRecorder(sql, [self.connector], config_path, strategy_name) + recorder.start() + + try: + # Try to buy some token from the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.05") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + order_id = self._place_order(True, amount, OrderType.LIMIT, price, 1, None, + fixture.WS_TRADE) + self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + + # Reset the logs + self.event_logger.clear() + + # Try to sell back the same amount to the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.95") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + order_id = self._place_order(False, amount, OrderType.LIMIT, price, 2, None, + fixture.WS_TRADE) + self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCompletedEvent)) + + # Query the persisted trade logs + trade_fills: List[TradeFill] = recorder.get_trades_for_config(config_path) + self.assertGreaterEqual(len(trade_fills), 2) + buy_fills: List[TradeFill] = [t for t in trade_fills if t.trade_type == "BUY"] + sell_fills: List[TradeFill] = [t for t in trade_fills if t.trade_type == "SELL"] + self.assertGreaterEqual(len(buy_fills), 1) + self.assertGreaterEqual(len(sell_fills), 1) + + order_id = None + + finally: + if order_id is not None: + self.connector.cancel(self.trading_pair, order_id) + self.run_parallel(self.event_logger.wait_for(OrderCancelledEvent)) + + recorder.stop() + # sql._engine.dispose() + # on windows cannot unlink the sqlite db file before closing the db + if os.name != 'nt': + os.unlink(self.db_path) + + +# unittest.main() diff --git a/test/connector/exchange/digifinex/test_digifinex_order_book_tracker.py b/test/connector/exchange/digifinex/test_digifinex_order_book_tracker.py new file mode 100644 index 0000000000..ebbe50fc0e --- /dev/null +++ b/test/connector/exchange/digifinex/test_digifinex_order_book_tracker.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +from os.path import join, realpath +import sys; sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +import math +import time +import asyncio +import logging +import unittest +from typing import Dict, Optional, List +from hummingbot.core.event.event_logger import EventLogger +from hummingbot.core.event.events import OrderBookEvent, OrderBookTradeEvent, TradeType +from hummingbot.connector.exchange.digifinex.digifinex_order_book_tracker import DigifinexOrderBookTracker +from hummingbot.connector.exchange.digifinex.digifinex_api_order_book_data_source import DigifinexAPIOrderBookDataSource +from hummingbot.core.data_type.order_book import OrderBook + + +class DigifinexOrderBookTrackerUnitTest(unittest.TestCase): + order_book_tracker: Optional[DigifinexOrderBookTracker] = None + events: List[OrderBookEvent] = [ + OrderBookEvent.TradeEvent + ] + trading_pairs: List[str] = [ + "BTC-USDT", + "ETH-USDT", + ] + + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + cls.order_book_tracker: DigifinexOrderBookTracker = DigifinexOrderBookTracker(cls.trading_pairs) + cls.order_book_tracker.start() + cls.ev_loop.run_until_complete(cls.wait_til_tracker_ready()) + + @classmethod + async def wait_til_tracker_ready(cls): + while True: + if len(cls.order_book_tracker.order_books) > 0: + print("Initialized real-time order books.") + return + await asyncio.sleep(1) + + async def run_parallel_async(self, *tasks, timeout=None): + logmsg = [f"{x.__qualname__}({x.cr_frame.f_locals if x.cr_frame is not None else None})" for x in tasks] + future: asyncio.Future = asyncio.ensure_future(asyncio.gather(*tasks)) + timer = 0 + while not future.done(): + if timeout and timer > timeout: + raise Exception("Timeout running parallel async tasks in tests") + timer += 1 + now = time.time() + _next_iteration = now // 1.0 + 1 # noqa: F841 + await asyncio.sleep(1.0) + if future._exception is not None: + logging.exception(logmsg) + return future.result() + + def run_parallel(self, *tasks): + return self.ev_loop.run_until_complete(self.run_parallel_async(*tasks)) + + def setUp(self): + self.event_logger = EventLogger() + for event_tag in self.events: + for trading_pair, order_book in self.order_book_tracker.order_books.items(): + order_book.add_listener(event_tag, self.event_logger) + + def test_order_book_trade_event_emission(self): + """ + Tests if the order book tracker is able to retrieve order book trade message from exchange and emit order book + trade events after correctly parsing the trade messages + """ + self.run_parallel(self.event_logger.wait_for(OrderBookTradeEvent)) + for ob_trade_event in self.event_logger.event_log: + self.assertTrue(type(ob_trade_event) == OrderBookTradeEvent) + self.assertTrue(ob_trade_event.trading_pair in self.trading_pairs) + self.assertTrue(type(ob_trade_event.timestamp) in [float, int]) + self.assertTrue(type(ob_trade_event.amount) == float) + self.assertTrue(type(ob_trade_event.price) == float) + self.assertTrue(type(ob_trade_event.type) == TradeType) + # datetime is in seconds + self.assertTrue(math.ceil(math.log10(ob_trade_event.timestamp)) == 10) + self.assertTrue(ob_trade_event.amount > 0) + self.assertTrue(ob_trade_event.price > 0) + + def test_tracker_integrity(self): + # Wait 5 seconds to process some diffs. + self.ev_loop.run_until_complete(asyncio.sleep(10.0)) + order_books: Dict[str, OrderBook] = self.order_book_tracker.order_books + eth_usdt: OrderBook = order_books["ETH-USDT"] + self.assertIsNot(eth_usdt.last_diff_uid, 0) + self.assertGreaterEqual(eth_usdt.get_price_for_volume(True, 10).result_price, + eth_usdt.get_price(True)) + self.assertLessEqual(eth_usdt.get_price_for_volume(False, 10).result_price, + eth_usdt.get_price(False)) + + def test_api_get_last_traded_prices(self): + prices = self.ev_loop.run_until_complete( + DigifinexAPIOrderBookDataSource.get_last_traded_prices(["BTC-USDT", "LTC-USDT"])) + for key, value in prices.items(): + print(f"{key} last_trade_price: {value}") + self.assertGreater(prices["BTC-USDT"], 1000) + self.assertLess(prices["LTC-USDT"], 10000) diff --git a/test/connector/exchange/digifinex/test_digifinex_user_stream_tracker.py b/test/connector/exchange/digifinex/test_digifinex_user_stream_tracker.py new file mode 100644 index 0000000000..f529ee22f6 --- /dev/null +++ b/test/connector/exchange/digifinex/test_digifinex_user_stream_tracker.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +from os.path import join, realpath +import sys; sys.path.insert(0, realpath(join(__file__, "../../../../../"))) + +import asyncio +import logging +import unittest +import conf + +from os.path import join, realpath +from hummingbot.connector.exchange.digifinex.digifinex_user_stream_tracker import DigifinexUserStreamTracker +from hummingbot.connector.exchange.digifinex.digifinex_global import DigifinexGlobal +from hummingbot.core.utils.async_utils import safe_ensure_future + +sys.path.insert(0, realpath(join(__file__, "../../../"))) + + +class DigifinexUserStreamTrackerUnitTest(unittest.TestCase): + api_key = conf.digifinex_api_key + api_secret = conf.digifinex_secret_key + + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + cls._global = DigifinexGlobal(cls.api_key, cls.api_secret) + cls.trading_pairs = ["BTC-USDT"] + cls.user_stream_tracker: DigifinexUserStreamTracker = DigifinexUserStreamTracker( + cls._global, trading_pairs=cls.trading_pairs) + cls.user_stream_tracker_task: asyncio.Task = safe_ensure_future(cls.user_stream_tracker.start()) + + def test_user_stream(self): + # Wait process some msgs. + self.ev_loop.run_until_complete(asyncio.sleep(120.0)) + print(self.user_stream_tracker.user_stream) + + +def main(): + logging.basicConfig(level=logging.INFO) + unittest.main() + + +if __name__ == "__main__": + main() diff --git a/test/connector/exchange/hitbtc/.gitignore b/test/connector/exchange/hitbtc/.gitignore new file mode 100644 index 0000000000..23d9952b8c --- /dev/null +++ b/test/connector/exchange/hitbtc/.gitignore @@ -0,0 +1 @@ +backups \ No newline at end of file diff --git a/test/connector/exchange/hitbtc/__init__.py b/test/connector/exchange/hitbtc/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/connector/exchange/hitbtc/test_hitbtc_auth.py b/test/connector/exchange/hitbtc/test_hitbtc_auth.py new file mode 100644 index 0000000000..1943412ea3 --- /dev/null +++ b/test/connector/exchange/hitbtc/test_hitbtc_auth.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +import sys +import asyncio +import unittest +import aiohttp +import conf +import logging +from os.path import join, realpath +from typing import Dict, Any +from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth +from hummingbot.connector.exchange.hitbtc.hitbtc_websocket import HitbtcWebsocket +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants + +sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +logging.basicConfig(level=METRICS_LOG_LEVEL) + + +class TestAuth(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + api_key = conf.hitbtc_api_key + secret_key = conf.hitbtc_secret_key + cls.auth = HitbtcAuth(api_key, secret_key) + + async def rest_auth(self) -> Dict[Any, Any]: + endpoint = Constants.ENDPOINT['USER_BALANCES'] + headers = self.auth.get_headers("GET", f"{Constants.REST_URL_AUTH}/{endpoint}", None) + http_client = aiohttp.ClientSession() + response = await http_client.get(f"{Constants.REST_URL}/{endpoint}", headers=headers) + await http_client.close() + return await response.json() + + async def ws_auth(self) -> Dict[Any, Any]: + ws = HitbtcWebsocket(self.auth) + await ws.connect() + await ws.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"], None, {}) + async for response in ws.on_message(): + return response + + def test_rest_auth(self): + result = self.ev_loop.run_until_complete(self.rest_auth()) + if len(result) == 0 or "currency" not in result[0].keys(): + print(f"Unexpected response for API call: {result}") + assert "currency" in result[0].keys() + + def test_ws_auth(self): + response = self.ev_loop.run_until_complete(self.ws_auth()) + if 'result' not in response: + print(f"Unexpected response for API call: {response}") + assert response['result'] is True diff --git a/test/connector/exchange/hitbtc/test_hitbtc_exchange.py b/test/connector/exchange/hitbtc/test_hitbtc_exchange.py new file mode 100644 index 0000000000..0456f5a8a9 --- /dev/null +++ b/test/connector/exchange/hitbtc/test_hitbtc_exchange.py @@ -0,0 +1,438 @@ +from os.path import join, realpath +import sys; sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +import asyncio +import logging +from decimal import Decimal +import unittest +import contextlib +import time +import os +from typing import List +import conf +import math + +from hummingbot.core.clock import Clock, ClockMode +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL +from hummingbot.core.utils.async_utils import safe_gather, safe_ensure_future +from hummingbot.core.event.event_logger import EventLogger +from hummingbot.core.event.events import ( + BuyOrderCompletedEvent, + BuyOrderCreatedEvent, + MarketEvent, + OrderFilledEvent, + OrderType, + SellOrderCompletedEvent, + SellOrderCreatedEvent, + OrderCancelledEvent +) +from hummingbot.model.sql_connection_manager import ( + SQLConnectionManager, + SQLConnectionType +) +from hummingbot.model.market_state import MarketState +from hummingbot.model.order import Order +from hummingbot.model.trade_fill import TradeFill +from hummingbot.connector.markets_recorder import MarketsRecorder +from hummingbot.connector.exchange.hitbtc.hitbtc_exchange import HitbtcExchange + +logging.basicConfig(level=METRICS_LOG_LEVEL) + +API_KEY = conf.hitbtc_api_key +API_SECRET = conf.hitbtc_secret_key + + +class HitbtcExchangeUnitTest(unittest.TestCase): + events: List[MarketEvent] = [ + MarketEvent.BuyOrderCompleted, + MarketEvent.SellOrderCompleted, + MarketEvent.OrderFilled, + MarketEvent.TransactionFailure, + MarketEvent.BuyOrderCreated, + MarketEvent.SellOrderCreated, + MarketEvent.OrderCancelled, + MarketEvent.OrderFailure + ] + connector: HitbtcExchange + event_logger: EventLogger + trading_pair = "BTC-USD" + base_token, quote_token = trading_pair.split("-") + stack: contextlib.ExitStack + + @classmethod + def setUpClass(cls): + global MAINNET_RPC_URL + + cls.ev_loop = asyncio.get_event_loop() + + cls.clock: Clock = Clock(ClockMode.REALTIME) + cls.connector: HitbtcExchange = HitbtcExchange( + hitbtc_api_key=API_KEY, + hitbtc_secret_key=API_SECRET, + trading_pairs=[cls.trading_pair], + trading_required=True + ) + print("Initializing Hitbtc market... this will take about a minute.") + cls.clock.add_iterator(cls.connector) + cls.stack: contextlib.ExitStack = contextlib.ExitStack() + cls._clock = cls.stack.enter_context(cls.clock) + cls.ev_loop.run_until_complete(cls.wait_til_ready()) + print("Ready.") + + @classmethod + def tearDownClass(cls) -> None: + cls.stack.close() + + @classmethod + async def wait_til_ready(cls, connector = None): + if connector is None: + connector = cls.connector + while True: + now = time.time() + next_iteration = now // 1.0 + 1 + if connector.ready: + break + else: + await cls._clock.run_til(next_iteration) + await asyncio.sleep(1.0) + + def setUp(self): + self.db_path: str = realpath(join(__file__, "../connector_test.sqlite")) + try: + os.unlink(self.db_path) + except FileNotFoundError: + pass + + self.event_logger = EventLogger() + for event_tag in self.events: + self.connector.add_listener(event_tag, self.event_logger) + + def tearDown(self): + for event_tag in self.events: + self.connector.remove_listener(event_tag, self.event_logger) + self.event_logger = None + + async def run_parallel_async(self, *tasks): + future: asyncio.Future = safe_ensure_future(safe_gather(*tasks)) + while not future.done(): + now = time.time() + next_iteration = now // 1.0 + 1 + await self._clock.run_til(next_iteration) + await asyncio.sleep(1.0) + return future.result() + + def run_parallel(self, *tasks): + return self.ev_loop.run_until_complete(self.run_parallel_async(*tasks)) + + def _place_order(self, is_buy, amount, order_type, price, ex_order_id) -> str: + if is_buy: + cl_order_id = self.connector.buy(self.trading_pair, amount, order_type, price) + else: + cl_order_id = self.connector.sell(self.trading_pair, amount, order_type, price) + return cl_order_id + + def _cancel_order(self, cl_order_id, connector=None): + if connector is None: + connector = self.connector + return connector.cancel(self.trading_pair, cl_order_id) + + def test_estimate_fee(self): + maker_fee = self.connector.estimate_fee_pct(True) + self.assertAlmostEqual(maker_fee, Decimal("0.001")) + taker_fee = self.connector.estimate_fee_pct(False) + self.assertAlmostEqual(taker_fee, Decimal("0.0025")) + + def test_buy_and_sell(self): + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.02") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + quote_bal = self.connector.get_available_balance(self.quote_token) + base_bal = self.connector.get_available_balance(self.base_token) + + order_id = self._place_order(True, amount, OrderType.LIMIT, price, 1) + order_completed_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(5)) + trade_events = [t for t in self.event_logger.event_log if isinstance(t, OrderFilledEvent)] + base_amount_traded = sum(t.amount for t in trade_events) + quote_amount_traded = sum(t.amount * t.price for t in trade_events) + + self.assertTrue([evt.order_type == OrderType.LIMIT for evt in trade_events]) + self.assertEqual(order_id, order_completed_event.order_id) + self.assertEqual(amount, order_completed_event.base_asset_amount) + self.assertEqual("BTC", order_completed_event.base_asset) + self.assertEqual("USD", order_completed_event.quote_asset) + self.assertAlmostEqual(base_amount_traded, order_completed_event.base_asset_amount) + self.assertAlmostEqual(quote_amount_traded, order_completed_event.quote_asset_amount) + self.assertGreater(order_completed_event.fee_amount, Decimal(0)) + self.assertTrue(any([isinstance(event, BuyOrderCreatedEvent) and str(event.order_id) == str(order_id) + for event in self.event_logger.event_log])) + + # check available quote balance gets updated, we need to wait a bit for the balance message to arrive + expected_quote_bal = quote_bal - quote_amount_traded + # self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.ev_loop.run_until_complete(self.connector._update_balances()) + self.assertAlmostEqual(expected_quote_bal, self.connector.get_available_balance(self.quote_token), 1) + + # Reset the logs + self.event_logger.clear() + + # Try to sell back the same amount to the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.98") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + order_id = self._place_order(False, amount, OrderType.LIMIT, price, 2) + order_completed_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCompletedEvent)) + trade_events = [t for t in self.event_logger.event_log if isinstance(t, OrderFilledEvent)] + base_amount_traded = sum(t.amount for t in trade_events) + quote_amount_traded = sum(t.amount * t.price for t in trade_events) + + self.assertTrue([evt.order_type == OrderType.LIMIT for evt in trade_events]) + self.assertEqual(order_id, order_completed_event.order_id) + self.assertEqual(amount, order_completed_event.base_asset_amount) + self.assertEqual("BTC", order_completed_event.base_asset) + self.assertEqual("USD", order_completed_event.quote_asset) + self.assertAlmostEqual(base_amount_traded, order_completed_event.base_asset_amount) + self.assertAlmostEqual(quote_amount_traded, order_completed_event.quote_asset_amount) + self.assertGreater(order_completed_event.fee_amount, Decimal(0)) + self.assertTrue(any([isinstance(event, SellOrderCreatedEvent) and event.order_id == order_id + for event in self.event_logger.event_log])) + + # check available base balance gets updated, we need to wait a bit for the balance message to arrive + expected_base_bal = base_bal + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.ev_loop.run_until_complete(self.connector._update_balances()) + self.ev_loop.run_until_complete(asyncio.sleep(5)) + self.assertAlmostEqual(expected_base_bal, self.connector.get_available_balance(self.base_token), 5) + + def test_limit_makers_unfilled(self): + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.8") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.ev_loop.run_until_complete(self.connector._update_balances()) + self.ev_loop.run_until_complete(asyncio.sleep(2)) + quote_bal = self.connector.get_available_balance(self.quote_token) + + cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + # check available quote balance gets updated, we need to wait a bit for the balance message to arrive + taker_fee = self.connector.estimate_fee_pct(False) + quote_amount = ((price * amount)) + quote_amount = ((price * amount) * (Decimal("1") + taker_fee)) + expected_quote_bal = quote_bal - quote_amount + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.ev_loop.run_until_complete(self.connector._update_balances()) + self.ev_loop.run_until_complete(asyncio.sleep(2)) + + self.assertAlmostEqual(expected_quote_bal, self.connector.get_available_balance(self.quote_token), 5) + self._cancel_order(cl_order_id) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.2") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + cl_order_id = self._place_order(False, amount, OrderType.LIMIT_MAKER, price, 2) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + self._cancel_order(cl_order_id) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + # # @TODO: find a way to create "rejected" + # def test_limit_maker_rejections(self): + # price = self.connector.get_price(self.trading_pair, True) * Decimal("1.2") + # price = self.connector.quantize_order_price(self.trading_pair, price) + # amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.000001")) + # cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1) + # event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + # self.assertEqual(cl_order_id, event.order_id) + + # price = self.connector.get_price(self.trading_pair, False) * Decimal("0.8") + # price = self.connector.quantize_order_price(self.trading_pair, price) + # amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.000001")) + # cl_order_id = self._place_order(False, amount, OrderType.LIMIT_MAKER, price, 2) + # event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + # self.assertEqual(cl_order_id, event.order_id) + + def test_cancel_all(self): + bid_price = self.connector.get_price(self.trading_pair, True) + ask_price = self.connector.get_price(self.trading_pair, False) + bid_price = self.connector.quantize_order_price(self.trading_pair, bid_price * Decimal("0.9")) + ask_price = self.connector.quantize_order_price(self.trading_pair, ask_price * Decimal("1.1")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + buy_id = self._place_order(True, amount, OrderType.LIMIT, bid_price, 1) + sell_id = self._place_order(False, amount, OrderType.LIMIT, ask_price, 2) + + self.ev_loop.run_until_complete(asyncio.sleep(1)) + asyncio.ensure_future(self.connector.cancel_all(5)) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + cancel_events = [t for t in self.event_logger.event_log if isinstance(t, OrderCancelledEvent)] + self.assertEqual({buy_id, sell_id}, {o.order_id for o in cancel_events}) + + def test_order_quantized_values(self): + bid_price: Decimal = self.connector.get_price(self.trading_pair, True) + ask_price: Decimal = self.connector.get_price(self.trading_pair, False) + mid_price: Decimal = (bid_price + ask_price) / 2 + + # Make sure there's enough balance to make the limit orders. + self.assertGreater(self.connector.get_balance("BTC"), Decimal("0.0005")) + self.assertGreater(self.connector.get_balance("USD"), Decimal("10")) + + # Intentionally set some prices with too many decimal places s.t. they + # need to be quantized. Also, place them far away from the mid-price s.t. they won't + # get filled during the test. + bid_price = self.connector.quantize_order_price(self.trading_pair, mid_price * Decimal("0.9333192292111341")) + ask_price = self.connector.quantize_order_price(self.trading_pair, mid_price * Decimal("1.1492431474884933")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.000123456")) + + # Test bid order + cl_order_id_1 = self._place_order(True, amount, OrderType.LIMIT, bid_price, 1) + # Wait for the order created event and examine the order made + self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + + # Test ask order + cl_order_id_2 = self._place_order(False, amount, OrderType.LIMIT, ask_price, 1) + # Wait for the order created event and examine and order made + self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCreatedEvent)) + + self._cancel_order(cl_order_id_1) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self._cancel_order(cl_order_id_2) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + + def test_orders_saving_and_restoration(self): + config_path = "test_config" + strategy_name = "test_strategy" + sql = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, db_path=self.db_path) + order_id = None + recorder = MarketsRecorder(sql, [self.connector], config_path, strategy_name) + recorder.start() + + try: + self.connector._in_flight_orders.clear() + self.assertEqual(0, len(self.connector.tracking_states)) + + # Try to put limit buy order for 0.02 ETH worth of ZRX, and watch for order creation event. + current_bid_price: Decimal = self.connector.get_price(self.trading_pair, True) + price: Decimal = current_bid_price * Decimal("0.8") + price = self.connector.quantize_order_price(self.trading_pair, price) + + amount: Decimal = Decimal("0.0001") + amount = self.connector.quantize_order_amount(self.trading_pair, amount) + + cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + + # Verify tracking states + self.assertEqual(1, len(self.connector.tracking_states)) + self.assertEqual(cl_order_id, list(self.connector.tracking_states.keys())[0]) + + # Verify orders from recorder + recorded_orders: List[Order] = recorder.get_orders_for_config_and_market(config_path, self.connector) + self.assertEqual(1, len(recorded_orders)) + self.assertEqual(cl_order_id, recorded_orders[0].id) + + # Verify saved market states + saved_market_states: MarketState = recorder.get_market_states(config_path, self.connector) + self.assertIsNotNone(saved_market_states) + self.assertIsInstance(saved_market_states.saved_state, dict) + self.assertGreater(len(saved_market_states.saved_state), 0) + + # Close out the current market and start another market. + self.connector.stop(self._clock) + self.ev_loop.run_until_complete(asyncio.sleep(5)) + self.clock.remove_iterator(self.connector) + for event_tag in self.events: + self.connector.remove_listener(event_tag, self.event_logger) + # Clear the event loop + self.event_logger.clear() + new_connector = HitbtcExchange(API_KEY, API_SECRET, [self.trading_pair], True) + for event_tag in self.events: + new_connector.add_listener(event_tag, self.event_logger) + recorder.stop() + recorder = MarketsRecorder(sql, [new_connector], config_path, strategy_name) + recorder.start() + saved_market_states = recorder.get_market_states(config_path, new_connector) + self.clock.add_iterator(new_connector) + self.ev_loop.run_until_complete(self.wait_til_ready(new_connector)) + self.assertEqual(0, len(new_connector.limit_orders)) + self.assertEqual(0, len(new_connector.tracking_states)) + new_connector.restore_tracking_states(saved_market_states.saved_state) + self.assertEqual(1, len(new_connector.limit_orders)) + self.assertEqual(1, len(new_connector.tracking_states)) + + # Cancel the order and verify that the change is saved. + self._cancel_order(cl_order_id, new_connector) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + recorder.save_market_states(config_path, new_connector) + order_id = None + self.assertEqual(0, len(new_connector.limit_orders)) + self.assertEqual(0, len(new_connector.tracking_states)) + saved_market_states = recorder.get_market_states(config_path, new_connector) + self.assertEqual(0, len(saved_market_states.saved_state)) + finally: + if order_id is not None: + self.connector.cancel(self.trading_pair, cl_order_id) + self.run_parallel(self.event_logger.wait_for(OrderCancelledEvent)) + + recorder.stop() + os.unlink(self.db_path) + + def test_update_last_prices(self): + # This is basic test to see if order_book last_trade_price is initiated and updated. + for order_book in self.connector.order_books.values(): + for _ in range(5): + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.assertFalse(math.isnan(order_book.last_trade_price)) + + def test_filled_orders_recorded(self): + config_path: str = "test_config" + strategy_name: str = "test_strategy" + sql = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, db_path=self.db_path) + order_id = None + recorder = MarketsRecorder(sql, [self.connector], config_path, strategy_name) + recorder.start() + + try: + # Try to buy some token from the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.05") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + order_id = self._place_order(True, amount, OrderType.LIMIT, price, 1) + self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + + # Reset the logs + self.event_logger.clear() + + # Try to sell back the same amount to the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.95") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + order_id = self._place_order(False, amount, OrderType.LIMIT, price, 2) + self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + + # Query the persisted trade logs + trade_fills: List[TradeFill] = recorder.get_trades_for_config(config_path) + self.assertGreaterEqual(len(trade_fills), 2) + buy_fills: List[TradeFill] = [t for t in trade_fills if t.trade_type == "BUY"] + sell_fills: List[TradeFill] = [t for t in trade_fills if t.trade_type == "SELL"] + self.assertGreaterEqual(len(buy_fills), 1) + self.assertGreaterEqual(len(sell_fills), 1) + + order_id = None + + finally: + if order_id is not None: + self.connector.cancel(self.trading_pair, order_id) + self.run_parallel(self.event_logger.wait_for(OrderCancelledEvent)) + + recorder.stop() + os.unlink(self.db_path) diff --git a/test/connector/exchange/hitbtc/test_hitbtc_order_book_tracker.py b/test/connector/exchange/hitbtc/test_hitbtc_order_book_tracker.py new file mode 100755 index 0000000000..ae3778e7c9 --- /dev/null +++ b/test/connector/exchange/hitbtc/test_hitbtc_order_book_tracker.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +import sys +import math +import time +import asyncio +import logging +import unittest +from os.path import join, realpath +from typing import Dict, Optional, List +from hummingbot.core.event.event_logger import EventLogger +from hummingbot.core.event.events import OrderBookEvent, OrderBookTradeEvent, TradeType +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_tracker import HitbtcOrderBookTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_api_order_book_data_source import HitbtcAPIOrderBookDataSource +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL + + +sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +logging.basicConfig(level=METRICS_LOG_LEVEL) + + +class HitbtcOrderBookTrackerUnitTest(unittest.TestCase): + order_book_tracker: Optional[HitbtcOrderBookTracker] = None + events: List[OrderBookEvent] = [ + OrderBookEvent.TradeEvent + ] + trading_pairs: List[str] = [ + "BTC-USD", + "ETH-USD", + ] + + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + cls.order_book_tracker: HitbtcOrderBookTracker = HitbtcOrderBookTracker(cls.trading_pairs) + cls.order_book_tracker.start() + cls.ev_loop.run_until_complete(cls.wait_til_tracker_ready()) + + @classmethod + async def wait_til_tracker_ready(cls): + while True: + if len(cls.order_book_tracker.order_books) > 0: + print("Initialized real-time order books.") + return + await asyncio.sleep(1) + + async def run_parallel_async(self, *tasks, timeout=None): + future: asyncio.Future = asyncio.ensure_future(asyncio.gather(*tasks)) + timer = 0 + while not future.done(): + if timeout and timer > timeout: + raise Exception("Timeout running parallel async tasks in tests") + timer += 1 + now = time.time() + _next_iteration = now // 1.0 + 1 # noqa: F841 + await asyncio.sleep(1.0) + return future.result() + + def run_parallel(self, *tasks): + return self.ev_loop.run_until_complete(self.run_parallel_async(*tasks)) + + def setUp(self): + self.event_logger = EventLogger() + for event_tag in self.events: + for trading_pair, order_book in self.order_book_tracker.order_books.items(): + order_book.add_listener(event_tag, self.event_logger) + + def test_order_book_trade_event_emission(self): + """ + Tests if the order book tracker is able to retrieve order book trade message from exchange and emit order book + trade events after correctly parsing the trade messages + """ + self.run_parallel(self.event_logger.wait_for(OrderBookTradeEvent)) + for ob_trade_event in self.event_logger.event_log: + self.assertTrue(type(ob_trade_event) == OrderBookTradeEvent) + self.assertTrue(ob_trade_event.trading_pair in self.trading_pairs) + self.assertTrue(type(ob_trade_event.timestamp) in [float, int]) + self.assertTrue(type(ob_trade_event.amount) == float) + self.assertTrue(type(ob_trade_event.price) == float) + self.assertTrue(type(ob_trade_event.type) == TradeType) + # datetime is in seconds + self.assertTrue(math.ceil(math.log10(ob_trade_event.timestamp)) == 10) + self.assertTrue(ob_trade_event.amount > 0) + self.assertTrue(ob_trade_event.price > 0) + + def test_tracker_integrity(self): + # Wait 5 seconds to process some diffs. + self.ev_loop.run_until_complete(asyncio.sleep(5.0)) + order_books: Dict[str, OrderBook] = self.order_book_tracker.order_books + eth_usd: OrderBook = order_books["ETH-USD"] + self.assertIsNot(eth_usd.last_diff_uid, 0) + self.assertGreaterEqual(eth_usd.get_price_for_volume(True, 10).result_price, + eth_usd.get_price(True)) + self.assertLessEqual(eth_usd.get_price_for_volume(False, 10).result_price, + eth_usd.get_price(False)) + + def test_api_get_last_traded_prices(self): + prices = self.ev_loop.run_until_complete( + HitbtcAPIOrderBookDataSource.get_last_traded_prices(["BTC-USD", "LTC-BTC"])) + for key, value in prices.items(): + print(f"{key} last_trade_price: {value}") + self.assertGreater(prices["BTC-USD"], 1000) + self.assertLess(prices["LTC-BTC"], 1) diff --git a/test/connector/exchange/hitbtc/test_hitbtc_user_stream_tracker.py b/test/connector/exchange/hitbtc/test_hitbtc_user_stream_tracker.py new file mode 100644 index 0000000000..5c82f2372b --- /dev/null +++ b/test/connector/exchange/hitbtc/test_hitbtc_user_stream_tracker.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +import sys +import asyncio +import logging +import unittest +import conf + +from os.path import join, realpath +from hummingbot.connector.exchange.hitbtc.hitbtc_user_stream_tracker import HitbtcUserStreamTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth +from hummingbot.core.utils.async_utils import safe_ensure_future +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL + + +sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +logging.basicConfig(level=METRICS_LOG_LEVEL) + + +class HitbtcUserStreamTrackerUnitTest(unittest.TestCase): + api_key = conf.hitbtc_api_key + api_secret = conf.hitbtc_secret_key + + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + cls.trading_pairs = ["BTC-USD"] + cls.user_stream_tracker: HitbtcUserStreamTracker = HitbtcUserStreamTracker( + hitbtc_auth=HitbtcAuth(cls.api_key, cls.api_secret), + trading_pairs=cls.trading_pairs) + cls.user_stream_tracker_task: asyncio.Task = safe_ensure_future(cls.user_stream_tracker.start()) + + def test_user_stream(self): + # Wait process some msgs. + print("Sleeping for 30s to gather some user stream messages.") + self.ev_loop.run_until_complete(asyncio.sleep(30.0)) + print(self.user_stream_tracker.user_stream) diff --git a/test/test_rate_oracle.py b/test/test_rate_oracle.py new file mode 100644 index 0000000000..9ecefae688 --- /dev/null +++ b/test/test_rate_oracle.py @@ -0,0 +1,61 @@ +import unittest +from decimal import Decimal +import asyncio +from hummingbot.core.rate_oracle.utils import find_rate +from hummingbot.core.rate_oracle.rate_oracle import RateOracle + + +class RateOracleTest(unittest.TestCase): + + def test_find_rate_from_source(self): + asyncio.get_event_loop().run_until_complete(self._test_find_rate_from_source()) + + async def _test_find_rate_from_source(self): + rate = await RateOracle.rate_async("BTC-USDT") + print(rate) + self.assertGreater(rate, 100) + + def test_get_rate_coingecko(self): + asyncio.get_event_loop().run_until_complete(self._test_get_rate_coingecko()) + + async def _test_get_rate_coingecko(self): + rates = await RateOracle.get_coingecko_prices_by_page("USD", 1) + print(rates) + self.assertGreater(len(rates), 100) + rates = await RateOracle.get_coingecko_prices("USD") + print(rates) + self.assertGreater(len(rates), 700) + + def test_rate_oracle_network(self): + oracle = RateOracle.get_instance() + oracle.start() + asyncio.get_event_loop().run_until_complete(oracle.get_ready()) + print(oracle.prices) + self.assertGreater(len(oracle.prices), 0) + rate = oracle.rate("SCRT-USDT") + print(f"rate SCRT-USDT: {rate}") + self.assertGreater(rate, 0) + rate1 = oracle.rate("BTC-USDT") + print(f"rate BTC-USDT: {rate1}") + self.assertGreater(rate1, 100) + # wait for 5 s to check rate again + asyncio.get_event_loop().run_until_complete(asyncio.sleep(5)) + rate2 = oracle.rate("BTC-USDT") + print(f"rate BTC-USDT: {rate2}") + self.assertNotEqual(0, rate2) + oracle.stop() + + def test_find_rate(self): + prices = {"HBOT-USDT": Decimal("100"), "AAVE-USDT": Decimal("50"), "USDT-GBP": Decimal("0.75")} + rate = find_rate(prices, "HBOT-USDT") + self.assertEqual(rate, Decimal("100")) + rate = find_rate(prices, "ZBOT-USDT") + self.assertEqual(rate, None) + rate = find_rate(prices, "USDT-HBOT") + self.assertEqual(rate, Decimal("0.01")) + rate = find_rate(prices, "HBOT-AAVE") + self.assertEqual(rate, Decimal("2")) + rate = find_rate(prices, "AAVE-HBOT") + self.assertEqual(rate, Decimal("0.5")) + rate = find_rate(prices, "HBOT-GBP") + self.assertEqual(rate, Decimal("75"))