From 3a83d296dbb27e6d53d9f113cc79552f73b0eb8e Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Mon, 1 Mar 2021 22:51:08 +0000 Subject: [PATCH 01/33] Initial HitBTC connector clone --- conf/__init__.py | 4 + hummingbot/connector/connector_status.py | 1 + .../connector/exchange/hitbtc/__init__.py | 0 .../hitbtc/hitbtc_active_order_tracker.pxd | 10 + .../hitbtc/hitbtc_active_order_tracker.pyx | 169 ++++ .../hitbtc_api_order_book_data_source.py | 221 +++++ .../hitbtc_api_user_stream_data_source.py | 76 ++ .../connector/exchange/hitbtc/hitbtc_auth.py | 58 ++ .../exchange/hitbtc/hitbtc_constants.py | 36 + .../exchange/hitbtc/hitbtc_exchange.py | 833 ++++++++++++++++++ .../exchange/hitbtc/hitbtc_in_flight_order.py | 99 +++ .../exchange/hitbtc/hitbtc_order_book.py | 146 +++ .../hitbtc/hitbtc_order_book_message.py | 80 ++ .../hitbtc/hitbtc_order_book_tracker.py | 110 +++ .../hitbtc/hitbtc_order_book_tracker_entry.py | 21 + .../hitbtc/hitbtc_user_stream_tracker.py | 73 ++ .../connector/exchange/hitbtc/hitbtc_utils.py | 90 ++ .../exchange/hitbtc/hitbtc_websocket.py | 128 +++ 18 files changed, 2155 insertions(+) create mode 100644 hummingbot/connector/exchange/hitbtc/__init__.py create mode 100644 hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pxd create mode 100644 hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx create mode 100644 hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py create mode 100755 hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py create mode 100755 hummingbot/connector/exchange/hitbtc/hitbtc_auth.py create mode 100644 hummingbot/connector/exchange/hitbtc/hitbtc_constants.py create mode 100644 hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py create mode 100644 hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py create mode 100644 hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py create mode 100644 hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py create mode 100644 hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py create mode 100644 hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker_entry.py create mode 100644 hummingbot/connector/exchange/hitbtc/hitbtc_user_stream_tracker.py create mode 100644 hummingbot/connector/exchange/hitbtc/hitbtc_utils.py create mode 100644 hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py diff --git a/conf/__init__.py b/conf/__init__.py index 65ba5a8347..7854c51a99 100644 --- a/conf/__init__.py +++ b/conf/__init__.py @@ -104,6 +104,10 @@ crypto_com_api_key = os.getenv("CRYPTO_COM_API_KEY") crypto_com_secret_key = os.getenv("CRYPTO_COM_SECRET_KEY") +# HitBTC Tests +hitbtc_api_key = os.getenv("HITBTC_API_KEY") +hitbtc_secret_key = os.getenv("HITBTC_SECRET_KEY") + # Wallet Tests test_erc20_token_address = os.getenv("TEST_ERC20_TOKEN_ADDRESS") web3_test_private_key_a = os.getenv("TEST_WALLET_PRIVATE_KEY_A") diff --git a/hummingbot/connector/connector_status.py b/hummingbot/connector/connector_status.py index f220808b1e..2f35999bb7 100644 --- a/hummingbot/connector/connector_status.py +++ b/hummingbot/connector/connector_status.py @@ -17,6 +17,7 @@ 'dydx': 'green', 'eterbase': 'red', 'ethereum': 'red', + 'hitbtc': 'yellow', 'huobi': 'green', 'kraken': 'green', 'kucoin': 'green', diff --git a/hummingbot/connector/exchange/hitbtc/__init__.py b/hummingbot/connector/exchange/hitbtc/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pxd b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pxd new file mode 100644 index 0000000000..68d8102dd0 --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pxd @@ -0,0 +1,10 @@ +# distutils: language=c++ +cimport numpy as np + +cdef class HitBTCActiveOrderTracker: + cdef dict _active_bids + cdef dict _active_asks + + cdef tuple c_convert_diff_message_to_np_arrays(self, object message) + cdef tuple c_convert_snapshot_message_to_np_arrays(self, object message) + cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx new file mode 100644 index 0000000000..a1fc15f519 --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx @@ -0,0 +1,169 @@ +# distutils: language=c++ +# distutils: sources=hummingbot/core/cpp/OrderBookEntry.cpp + +import logging +import numpy as np + +from decimal import Decimal +from typing import Dict +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.order_book_row import OrderBookRow + +_logger = None +s_empty_diff = np.ndarray(shape=(0, 4), dtype="float64") +HitBTCOrderBookTrackingDictionary = Dict[Decimal, Dict[str, Dict[str, any]]] + +cdef class HitBTCActiveOrderTracker: + def __init__(self, + active_asks: HitBTCOrderBookTrackingDictionary = None, + active_bids: HitBTCOrderBookTrackingDictionary = None): + super().__init__() + self._active_asks = active_asks or {} + self._active_bids = active_bids or {} + + @classmethod + def logger(cls) -> HummingbotLogger: + global _logger + if _logger is None: + _logger = logging.getLogger(__name__) + return _logger + + @property + def active_asks(self) -> HitBTCOrderBookTrackingDictionary: + return self._active_asks + + @property + def active_bids(self) -> HitBTCOrderBookTrackingDictionary: + return self._active_bids + + # TODO: research this more + def volume_for_ask_price(self, price) -> float: + return NotImplementedError + + # TODO: research this more + def volume_for_bid_price(self, price) -> float: + return NotImplementedError + + def get_rates_and_quantities(self, entry) -> tuple: + # price, quantity + return float(entry[0]), float(entry[1]) + + cdef tuple c_convert_diff_message_to_np_arrays(self, object message): + cdef: + dict content = message.content + list bid_entries = [] + list ask_entries = [] + str order_id + str order_side + str price_raw + object price + dict order_dict + double timestamp = message.timestamp + double amount = 0 + + bid_entries = content["bids"] + ask_entries = content["asks"] + + bids = s_empty_diff + asks = s_empty_diff + + if len(bid_entries) > 0: + bids = np.array( + [[timestamp, + float(price), + float(amount), + message.update_id] + for price, amount in [self.get_rates_and_quantities(entry) for entry in bid_entries]], + dtype="float64", + ndmin=2 + ) + + if len(ask_entries) > 0: + asks = np.array( + [[timestamp, + float(price), + float(amount), + message.update_id] + for price, amount in [self.get_rates_and_quantities(entry) for entry in ask_entries]], + dtype="float64", + ndmin=2 + ) + + return bids, asks + + cdef tuple c_convert_snapshot_message_to_np_arrays(self, object message): + cdef: + float price + float amount + str order_id + dict order_dict + + # Refresh all order tracking. + self._active_bids.clear() + self._active_asks.clear() + timestamp = message.timestamp + content = message.content + + for snapshot_orders, active_orders in [(content["bids"], self._active_bids), (content["asks"], self.active_asks)]: + for order in snapshot_orders: + price, amount = self.get_rates_and_quantities(order) + + order_dict = { + "order_id": timestamp, + "amount": amount + } + + if price in active_orders: + active_orders[price][timestamp] = order_dict + else: + active_orders[price] = { + timestamp: order_dict + } + + cdef: + np.ndarray[np.float64_t, ndim=2] bids = np.array( + [[message.timestamp, + price, + sum([order_dict["amount"] + for order_dict in self._active_bids[price].values()]), + message.update_id] + for price in sorted(self._active_bids.keys(), reverse=True)], dtype="float64", ndmin=2) + np.ndarray[np.float64_t, ndim=2] asks = np.array( + [[message.timestamp, + price, + sum([order_dict["amount"] + for order_dict in self.active_asks[price].values()]), + message.update_id] + for price in sorted(self.active_asks.keys(), reverse=True)], dtype="float64", ndmin=2 + ) + + if bids.shape[1] != 4: + bids = bids.reshape((0, 4)) + if asks.shape[1] != 4: + asks = asks.reshape((0, 4)) + + return bids, asks + + cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message): + cdef: + double trade_type_value = 2.0 + + timestamp = message.timestamp + content = message.content + + return np.array( + [timestamp, trade_type_value, float(content["price"]), float(content["size"])], + dtype="float64" + ) + + def convert_diff_message_to_order_book_row(self, message): + np_bids, np_asks = self.c_convert_diff_message_to_np_arrays(message) + bids_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_bids] + asks_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_asks] + return bids_row, asks_row + + def convert_snapshot_message_to_order_book_row(self, message): + np_bids, np_asks = self.c_convert_snapshot_message_to_np_arrays(message) + bids_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_bids] + asks_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_asks] + return bids_row, asks_row diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py new file mode 100644 index 0000000000..d803be91e1 --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python +import asyncio +import logging +import time +import aiohttp +import pandas as pd +import hummingbot.connector.exchange.hitbtc.hitbtc_constants as constants + +from typing import Optional, List, Dict, Any +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_message import OrderBookMessage +from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource +from hummingbot.core.utils.async_utils import safe_gather +from hummingbot.logger import HummingbotLogger +from . import hitbtc_utils +from .hitbtc_active_order_tracker import HitBTCActiveOrderTracker +from .hitbtc_order_book import HitBTCOrderBook +from .hitbtc_websocket import HitBTCWebsocket +from .hitbtc_utils import ms_timestamp_to_s + + +class HitBTCAPIOrderBookDataSource(OrderBookTrackerDataSource): + MAX_RETRIES = 20 + MESSAGE_TIMEOUT = 30.0 + SNAPSHOT_TIMEOUT = 10.0 + + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, trading_pairs: List[str] = None): + super().__init__(trading_pairs) + self._trading_pairs: List[str] = trading_pairs + self._snapshot_msg: Dict[str, any] = {} + + @classmethod + async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, float]: + result = {} + async with aiohttp.ClientSession() as client: + resp = await client.get(f"{constants.REST_URL}/public/get-ticker") + resp_json = await resp.json() + for t_pair in trading_pairs: + last_trade = [o["a"] for o in resp_json["result"]["data"] if o["i"] == + hitbtc_utils.convert_to_exchange_trading_pair(t_pair)] + if last_trade and last_trade[0] is not None: + result[t_pair] = last_trade[0] + return result + + @staticmethod + async def fetch_trading_pairs() -> List[str]: + async with aiohttp.ClientSession() as client: + async with client.get(f"{constants.REST_URL}/public/get-ticker", timeout=10) as response: + if response.status == 200: + from hummingbot.connector.exchange.hitbtc.hitbtc_utils import \ + convert_from_exchange_trading_pair + try: + data: Dict[str, Any] = await response.json() + return [convert_from_exchange_trading_pair(item["i"]) for item in data["result"]["data"]] + except Exception: + pass + # Do nothing if the request fails -- there will be no autocomplete for kucoin trading pairs + return [] + + @staticmethod + async def get_order_book_data(trading_pair: str) -> Dict[str, any]: + """ + Get whole orderbook + """ + async with aiohttp.ClientSession() as client: + orderbook_response = await client.get( + f"{constants.REST_URL}/public/get-book?depth=150&instrument_name=" + f"{hitbtc_utils.convert_to_exchange_trading_pair(trading_pair)}" + ) + + if orderbook_response.status != 200: + raise IOError( + f"Error fetching OrderBook for {trading_pair} at {constants.EXCHANGE_NAME}. " + f"HTTP status is {orderbook_response.status}." + ) + + orderbook_data: List[Dict[str, Any]] = await safe_gather(orderbook_response.json()) + orderbook_data = orderbook_data[0]["result"]["data"][0] + + return orderbook_data + + async def get_new_order_book(self, trading_pair: str) -> OrderBook: + snapshot: Dict[str, Any] = await self.get_order_book_data(trading_pair) + snapshot_timestamp: float = time.time() + snapshot_msg: OrderBookMessage = HitBTCOrderBook.snapshot_message_from_exchange( + snapshot, + snapshot_timestamp, + metadata={"trading_pair": trading_pair} + ) + order_book = self.order_book_create_function() + active_order_tracker: HitBTCActiveOrderTracker = HitBTCActiveOrderTracker() + bids, asks = active_order_tracker.convert_snapshot_message_to_order_book_row(snapshot_msg) + order_book.apply_snapshot(bids, asks, snapshot_msg.update_id) + return order_book + + async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for trades using websocket trade channel + """ + while True: + try: + ws = HitBTCWebsocket() + await ws.connect() + + await ws.subscribe(list(map( + lambda pair: f"trade.{hitbtc_utils.convert_to_exchange_trading_pair(pair)}", + self._trading_pairs + ))) + + async for response in ws.on_message(): + if response.get("result") is None: + continue + + for trade in response["result"]["data"]: + trade: Dict[Any] = trade + trade_timestamp: int = ms_timestamp_to_s(trade["t"]) + trade_msg: OrderBookMessage = HitBTCOrderBook.trade_message_from_exchange( + trade, + trade_timestamp, + metadata={"trading_pair": hitbtc_utils.convert_from_exchange_trading_pair(trade["i"])} + ) + output.put_nowait(trade_msg) + + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error.", exc_info=True) + await asyncio.sleep(5.0) + finally: + await ws.disconnect() + + async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for orderbook diffs using websocket book channel + """ + while True: + try: + ws = HitBTCWebsocket() + await ws.connect() + + await ws.subscribe(list(map( + lambda pair: f"book.{hitbtc_utils.convert_to_exchange_trading_pair(pair)}.150", + self._trading_pairs + ))) + + async for response in ws.on_message(): + if response.get("result") is None: + continue + + order_book_data = response["result"]["data"][0] + timestamp: int = ms_timestamp_to_s(order_book_data["t"]) + # data in this channel is not order book diff but the entire order book (up to depth 150). + # so we need to convert it into a order book snapshot. + # HitBTC does not offer order book diff ws updates. + orderbook_msg: OrderBookMessage = HitBTCOrderBook.snapshot_message_from_exchange( + order_book_data, + timestamp, + metadata={"trading_pair": hitbtc_utils.convert_from_exchange_trading_pair( + response["result"]["instrument_name"])} + ) + output.put_nowait(orderbook_msg) + + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unexpected error with WebSocket connection.", + exc_info=True, + app_warning_msg="Unexpected error with WebSocket connection. Retrying in 30 seconds. " + "Check network connection." + ) + await asyncio.sleep(30.0) + finally: + await ws.disconnect() + + async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for orderbook snapshots by fetching orderbook + """ + while True: + try: + for trading_pair in self._trading_pairs: + try: + snapshot: Dict[str, any] = await self.get_order_book_data(trading_pair) + snapshot_timestamp: int = ms_timestamp_to_s(snapshot["t"]) + snapshot_msg: OrderBookMessage = HitBTCOrderBook.snapshot_message_from_exchange( + snapshot, + snapshot_timestamp, + metadata={"trading_pair": trading_pair} + ) + output.put_nowait(snapshot_msg) + self.logger().debug(f"Saved order book snapshot for {trading_pair}") + # Be careful not to go above API rate limits. + await asyncio.sleep(5.0) + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unexpected error with WebSocket connection.", + exc_info=True, + app_warning_msg="Unexpected error with WebSocket connection. Retrying in 5 seconds. " + "Check network connection." + ) + await asyncio.sleep(5.0) + this_hour: pd.Timestamp = pd.Timestamp.utcnow().replace(minute=0, second=0, microsecond=0) + next_hour: pd.Timestamp = this_hour + pd.Timedelta(hours=1) + delta: float = next_hour.timestamp() - time.time() + await asyncio.sleep(delta) + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error.", exc_info=True) + await asyncio.sleep(5.0) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py new file mode 100755 index 0000000000..0c0e441788 --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +import time +import asyncio +import logging +from typing import Optional, List, AsyncIterable, Any +from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource +from hummingbot.logger import HummingbotLogger +from .hitbtc_auth import HitBTCAuth +from .hitbtc_websocket import HitBTCWebsocket + + +class HitBTCAPIUserStreamDataSource(UserStreamTrackerDataSource): + MAX_RETRIES = 20 + MESSAGE_TIMEOUT = 30.0 + + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, hitbtc_auth: HitBTCAuth, trading_pairs: Optional[List[str]] = []): + self._hitbtc_auth: HitBTCAuth = hitbtc_auth + self._trading_pairs = trading_pairs + self._current_listen_key = None + self._listen_for_user_stream_task = None + self._last_recv_time: float = 0 + super().__init__() + + @property + def last_recv_time(self) -> float: + return self._last_recv_time + + async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: + """ + Subscribe to active orders via web socket + """ + + try: + ws = HitBTCWebsocket(self._hitbtc_auth) + await ws.connect() + await ws.subscribe(["user.order", "user.trade", "user.balance"]) + async for msg in ws.on_message(): + # print(f"WS_SOCKET: {msg}") + yield msg + self._last_recv_time = time.time() + if (msg.get("result") is None): + continue + except Exception as e: + raise e + finally: + await ws.disconnect() + await asyncio.sleep(5) + + async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue) -> AsyncIterable[Any]: + """ + *required + Subscribe to user stream via web socket, and keep the connection open for incoming messages + :param ev_loop: ev_loop to execute this function in + :param output: an async queue where the incoming messages are stored + """ + + while True: + try: + async for msg in self._listen_to_orders_trades_balances(): + output.put_nowait(msg) + except asyncio.CancelledError: + raise + except Exception: + self.logger().error( + "Unexpected error with HitBTC WebSocket connection. " "Retrying after 30 seconds...", exc_info=True + ) + await asyncio.sleep(30.0) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py b/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py new file mode 100755 index 0000000000..abd6be087e --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py @@ -0,0 +1,58 @@ +import hmac +import hashlib +from typing import Dict, Any + + +class HitBTCAuth(): + """ + Auth class required by HitBTC API + Learn more at https://exchange-docs.crypto.com/#digital-signature + """ + def __init__(self, api_key: str, secret_key: str): + self.api_key = api_key + self.secret_key = secret_key + + def generate_auth_dict( + self, + path_url: str, + request_id: int, + nonce: int, + data: Dict[str, Any] = None + ): + """ + Generates authentication signature and return it in a dictionary along with other inputs + :return: a dictionary of request info including the request signature + """ + + data = data or {} + data['method'] = path_url + data.update({'nonce': nonce, 'api_key': self.api_key, 'id': request_id}) + + data_params = data.get('params', {}) + if not data_params: + data['params'] = {} + params = ''.join( + f'{key}{data_params[key]}' + for key in sorted(data_params) + ) + + payload = f"{path_url}{data['id']}" \ + f"{self.api_key}{params}{data['nonce']}" + + data['sig'] = hmac.new( + self.secret_key.encode('utf-8'), + payload.encode('utf-8'), + hashlib.sha256 + ).hexdigest() + + return data + + def get_headers(self) -> Dict[str, Any]: + """ + Generates authentication headers required by HitBTC + :return: a dictionary of auth headers + """ + + return { + "Content-Type": 'application/json', + } diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py new file mode 100644 index 0000000000..fe63df019e --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py @@ -0,0 +1,36 @@ +# A single source of truth for constant variables related to the exchange + + +EXCHANGE_NAME = "hitbtc" +REST_URL = "https://api.crypto.com/v2" +# WSS_PRIVATE_URL = "wss://stream.crypto.com/v2/user" +WSS_PRIVATE_URL = "wss://d289dek49b4wqs.cloudfront.net/v2/user" +# WSS_PUBLIC_URL = "wss://stream.crypto.com/v2/market" +WSS_PUBLIC_URL = "wss://d10tq1f9ygdz7y.cloudfront.net/v2/market" + +API_REASONS = { + 0: "Success", + 10001: "Malformed request, (E.g. not using application/json for REST)", + 10002: "Not authenticated, or key/signature incorrect", + 10003: "IP address not whitelisted", + 10004: "Missing required fields", + 10005: "Disallowed based on user tier", + 10006: "Requests have exceeded rate limits", + 10007: "Nonce value differs by more than 30 seconds from server", + 10008: "Invalid method specified", + 10009: "Invalid date range", + 20001: "Duplicated record", + 20002: "Insufficient balance", + 30003: "Invalid instrument_name specified", + 30004: "Invalid side specified", + 30005: "Invalid type specified", + 30006: "Price is lower than the minimum", + 30007: "Price is higher than the maximum", + 30008: "Quantity is lower than the minimum", + 30009: "Quantity is higher than the maximum", + 30010: "Required argument is blank or missing", + 30013: "Too many decimal places for Price", + 30014: "Too many decimal places for Quantity", + 30016: "The notional amount is less than the minimum", + 30017: "The notional amount exceeds the maximum", +} diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py new file mode 100644 index 0000000000..835cf1d4a4 --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -0,0 +1,833 @@ +import logging +from typing import ( + Dict, + List, + Optional, + Any, + AsyncIterable, +) +from decimal import Decimal +import asyncio +import json +import aiohttp +import math +import time + +from hummingbot.core.network_iterator import NetworkStatus +from hummingbot.logger import HummingbotLogger +from hummingbot.core.clock import Clock +from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather +from hummingbot.connector.trading_rule import TradingRule +from hummingbot.core.data_type.cancellation_result import CancellationResult +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.limit_order import LimitOrder +from hummingbot.core.event.events import ( + MarketEvent, + BuyOrderCompletedEvent, + SellOrderCompletedEvent, + OrderFilledEvent, + OrderCancelledEvent, + BuyOrderCreatedEvent, + SellOrderCreatedEvent, + MarketOrderFailureEvent, + OrderType, + TradeType, + TradeFee +) +from hummingbot.connector.exchange_base import ExchangeBase +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_tracker import HitBTCOrderBookTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_user_stream_tracker import HitBTCUserStreamTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitBTCAuth +from hummingbot.connector.exchange.hitbtc.hitbtc_in_flight_order import HitBTCInFlightOrder +from hummingbot.connector.exchange.hitbtc import hitbtc_utils +from hummingbot.connector.exchange.hitbtc import hitbtc_constants as Constants +from hummingbot.core.data_type.common import OpenOrder +ctce_logger = None +s_decimal_NaN = Decimal("nan") + + +class HitBTCExchange(ExchangeBase): + """ + HitBTCExchange connects with HitBTC exchange and provides order book pricing, user account tracking and + trading functionality. + """ + API_CALL_TIMEOUT = 10.0 + SHORT_POLL_INTERVAL = 5.0 + UPDATE_ORDER_STATUS_MIN_INTERVAL = 10.0 + LONG_POLL_INTERVAL = 120.0 + + @classmethod + def logger(cls) -> HummingbotLogger: + global ctce_logger + if ctce_logger is None: + ctce_logger = logging.getLogger(__name__) + return ctce_logger + + def __init__(self, + hitbtc_api_key: str, + hitbtc_secret_key: str, + trading_pairs: Optional[List[str]] = None, + trading_required: bool = True + ): + """ + :param hitbtc_api_key: The API key to connect to private HitBTC APIs. + :param hitbtc_secret_key: The API secret. + :param trading_pairs: The market trading pairs which to track order book data. + :param trading_required: Whether actual trading is needed. + """ + super().__init__() + self._trading_required = trading_required + self._trading_pairs = trading_pairs + self._hitbtc_auth = HitBTCAuth(hitbtc_api_key, hitbtc_secret_key) + self._order_book_tracker = HitBTCOrderBookTracker(trading_pairs=trading_pairs) + self._user_stream_tracker = HitBTCUserStreamTracker(self._hitbtc_auth, trading_pairs) + self._ev_loop = asyncio.get_event_loop() + self._shared_client = None + self._poll_notifier = asyncio.Event() + self._last_timestamp = 0 + self._in_flight_orders = {} # Dict[client_order_id:str, HitBTCInFlightOrder] + self._order_not_found_records = {} # Dict[client_order_id:str, count:int] + self._trading_rules = {} # Dict[trading_pair:str, TradingRule] + self._status_polling_task = None + self._user_stream_event_listener_task = None + self._trading_rules_polling_task = None + self._last_poll_timestamp = 0 + + @property + def name(self) -> str: + return "hitbtc" + + @property + def order_books(self) -> Dict[str, OrderBook]: + return self._order_book_tracker.order_books + + @property + def trading_rules(self) -> Dict[str, TradingRule]: + return self._trading_rules + + @property + def in_flight_orders(self) -> Dict[str, HitBTCInFlightOrder]: + return self._in_flight_orders + + @property + def status_dict(self) -> Dict[str, bool]: + """ + A dictionary of statuses of various connector's components. + """ + return { + "order_books_initialized": self._order_book_tracker.ready, + "account_balance": len(self._account_balances) > 0 if self._trading_required else True, + "trading_rule_initialized": len(self._trading_rules) > 0, + "user_stream_initialized": + self._user_stream_tracker.data_source.last_recv_time > 0 if self._trading_required else True, + } + + @property + def ready(self) -> bool: + """ + :return True when all statuses pass, this might take 5-10 seconds for all the connector's components and + services to be ready. + """ + return all(self.status_dict.values()) + + @property + def limit_orders(self) -> List[LimitOrder]: + return [ + in_flight_order.to_limit_order() + for in_flight_order in self._in_flight_orders.values() + ] + + @property + def tracking_states(self) -> Dict[str, any]: + """ + :return active in-flight orders in json format, is used to save in sqlite db. + """ + return { + key: value.to_json() + for key, value in self._in_flight_orders.items() + if not value.is_done + } + + def restore_tracking_states(self, saved_states: Dict[str, any]): + """ + Restore in-flight orders from saved tracking states, this is st the connector can pick up on where it left off + when it disconnects. + :param saved_states: The saved tracking_states. + """ + self._in_flight_orders.update({ + key: HitBTCInFlightOrder.from_json(value) + for key, value in saved_states.items() + }) + + def supported_order_types(self) -> List[OrderType]: + """ + :return a list of OrderType supported by this connector. + Note that Market order type is no longer required and will not be used. + """ + return [OrderType.LIMIT, OrderType.LIMIT_MAKER] + + def start(self, clock: Clock, timestamp: float): + """ + This function is called automatically by the clock. + """ + super().start(clock, timestamp) + + def stop(self, clock: Clock): + """ + This function is called automatically by the clock. + """ + super().stop(clock) + + async def start_network(self): + """ + This function is required by NetworkIterator base class and is called automatically. + It starts tracking order book, polling trading rules, + updating statuses and tracking user data. + """ + self._order_book_tracker.start() + self._trading_rules_polling_task = safe_ensure_future(self._trading_rules_polling_loop()) + if self._trading_required: + self._status_polling_task = safe_ensure_future(self._status_polling_loop()) + self._user_stream_tracker_task = safe_ensure_future(self._user_stream_tracker.start()) + self._user_stream_event_listener_task = safe_ensure_future(self._user_stream_event_listener()) + + async def stop_network(self): + """ + This function is required by NetworkIterator base class and is called automatically. + """ + self._order_book_tracker.stop() + if self._status_polling_task is not None: + self._status_polling_task.cancel() + self._status_polling_task = None + if self._trading_rules_polling_task is not None: + self._trading_rules_polling_task.cancel() + self._trading_rules_polling_task = None + if self._status_polling_task is not None: + self._status_polling_task.cancel() + self._status_polling_task = None + if self._user_stream_tracker_task is not None: + self._user_stream_tracker_task.cancel() + self._user_stream_tracker_task = None + if self._user_stream_event_listener_task is not None: + self._user_stream_event_listener_task.cancel() + self._user_stream_event_listener_task = None + + async def check_network(self) -> NetworkStatus: + """ + This function is required by NetworkIterator base class and is called periodically to check + the network connection. Simply ping the network (or call any light weight public API). + """ + try: + # since there is no ping endpoint, the lowest rate call is to get BTC-USDT ticker + await self._api_request("get", "public/get-ticker?instrument_name=BTC_USDT") + except asyncio.CancelledError: + raise + except Exception: + return NetworkStatus.NOT_CONNECTED + return NetworkStatus.CONNECTED + + async def _http_client(self) -> aiohttp.ClientSession: + """ + :returns Shared client session instance + """ + if self._shared_client is None: + self._shared_client = aiohttp.ClientSession() + return self._shared_client + + async def _trading_rules_polling_loop(self): + """ + Periodically update trading rule. + """ + while True: + try: + await self._update_trading_rules() + await asyncio.sleep(60) + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().network(f"Unexpected error while fetching trading rules. Error: {str(e)}", + exc_info=True, + app_warning_msg="Could not fetch new trading rules from HitBTC. " + "Check network connection.") + await asyncio.sleep(0.5) + + async def _update_trading_rules(self): + instruments_info = await self._api_request("get", path_url="public/get-instruments") + self._trading_rules.clear() + self._trading_rules = self._format_trading_rules(instruments_info) + + def _format_trading_rules(self, instruments_info: Dict[str, Any]) -> Dict[str, TradingRule]: + """ + Converts json API response into a dictionary of trading rules. + :param instruments_info: The json API response + :return A dictionary of trading rules. + Response Example: + { + "id": 11, + "method": "public/get-instruments", + "code": 0, + "result": { + "instruments": [ + { + "instrument_name": "ETH_CRO", + "quote_currency": "CRO", + "base_currency": "ETH", + "price_decimals": 2, + "quantity_decimals": 2 + }, + { + "instrument_name": "CRO_BTC", + "quote_currency": "BTC", + "base_currency": "CRO", + "price_decimals": 8, + "quantity_decimals": 2 + } + ] + } + } + """ + result = {} + for rule in instruments_info["result"]["instruments"]: + try: + trading_pair = hitbtc_utils.convert_from_exchange_trading_pair(rule["instrument_name"]) + price_decimals = Decimal(str(rule["price_decimals"])) + quantity_decimals = Decimal(str(rule["quantity_decimals"])) + # E.g. a price decimal of 2 means 0.01 incremental. + price_step = Decimal("1") / Decimal(str(math.pow(10, price_decimals))) + quantity_step = Decimal("1") / Decimal(str(math.pow(10, quantity_decimals))) + result[trading_pair] = TradingRule(trading_pair, + min_price_increment=price_step, + min_base_amount_increment=quantity_step) + except Exception: + self.logger().error(f"Error parsing the trading pair rule {rule}. Skipping.", exc_info=True) + return result + + async def _api_request(self, + method: str, + path_url: str, + params: Dict[str, Any] = {}, + is_auth_required: bool = False) -> Dict[str, Any]: + """ + Sends an aiohttp request and waits for a response. + :param method: The HTTP method, e.g. get or post + :param path_url: The path url or the API end point + :param is_auth_required: Whether an authentication is required, when True the function will add encrypted + signature to the request. + :returns A response in json format. + """ + url = f"{Constants.REST_URL}/{path_url}" + client = await self._http_client() + if is_auth_required: + request_id = hitbtc_utils.RequestId.generate_request_id() + data = {"params": params} + params = self._hitbtc_auth.generate_auth_dict(path_url, request_id, + hitbtc_utils.get_ms_timestamp(), data) + headers = self._hitbtc_auth.get_headers() + else: + headers = {"Content-Type": "application/json"} + + if method == "get": + response = await client.get(url, headers=headers) + elif method == "post": + post_json = json.dumps(params) + response = await client.post(url, data=post_json, headers=headers) + else: + raise NotImplementedError + + try: + parsed_response = json.loads(await response.text()) + except Exception as e: + raise IOError(f"Error parsing data from {url}. Error: {str(e)}") + if response.status != 200: + raise IOError(f"Error fetching data from {url}. HTTP status is {response.status}. " + f"Message: {parsed_response}") + if parsed_response["code"] != 0: + raise IOError(f"{url} API call failed, response: {parsed_response}") + # print(f"REQUEST: {method} {path_url} {params}") + # print(f"RESPONSE: {parsed_response}") + return parsed_response + + def get_order_price_quantum(self, trading_pair: str, price: Decimal): + """ + Returns a price step, a minimum price increment for a given trading pair. + """ + trading_rule = self._trading_rules[trading_pair] + return trading_rule.min_price_increment + + def get_order_size_quantum(self, trading_pair: str, order_size: Decimal): + """ + Returns an order amount step, a minimum amount increment for a given trading pair. + """ + trading_rule = self._trading_rules[trading_pair] + return Decimal(trading_rule.min_base_amount_increment) + + def get_order_book(self, trading_pair: str) -> OrderBook: + if trading_pair not in self._order_book_tracker.order_books: + raise ValueError(f"No order book exists for '{trading_pair}'.") + return self._order_book_tracker.order_books[trading_pair] + + def buy(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, + price: Decimal = s_decimal_NaN, **kwargs) -> str: + """ + Buys an amount of base asset (of the given trading pair). This function returns immediately. + To see an actual order, you'll have to wait for BuyOrderCreatedEvent. + :param trading_pair: The market (e.g. BTC-USDT) to buy from + :param amount: The amount in base token value + :param order_type: The order type + :param price: The price (note: this is no longer optional) + :returns A new internal order id + """ + order_id: str = hitbtc_utils.get_new_client_order_id(True, trading_pair) + safe_ensure_future(self._create_order(TradeType.BUY, order_id, trading_pair, amount, order_type, price)) + return order_id + + def sell(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, + price: Decimal = s_decimal_NaN, **kwargs) -> str: + """ + Sells an amount of base asset (of the given trading pair). This function returns immediately. + To see an actual order, you'll have to wait for SellOrderCreatedEvent. + :param trading_pair: The market (e.g. BTC-USDT) to sell from + :param amount: The amount in base token value + :param order_type: The order type + :param price: The price (note: this is no longer optional) + :returns A new internal order id + """ + order_id: str = hitbtc_utils.get_new_client_order_id(False, trading_pair) + safe_ensure_future(self._create_order(TradeType.SELL, order_id, trading_pair, amount, order_type, price)) + return order_id + + def cancel(self, trading_pair: str, order_id: str): + """ + Cancel an order. This function returns immediately. + To get the cancellation result, you'll have to wait for OrderCancelledEvent. + :param trading_pair: The market (e.g. BTC-USDT) of the order. + :param order_id: The internal order id (also called client_order_id) + """ + safe_ensure_future(self._execute_cancel(trading_pair, order_id)) + return order_id + + async def _create_order(self, + trade_type: TradeType, + order_id: str, + trading_pair: str, + amount: Decimal, + order_type: OrderType, + price: Decimal): + """ + Calls create-order API end point to place an order, starts tracking the order and triggers order created event. + :param trade_type: BUY or SELL + :param order_id: Internal order id (also called client_order_id) + :param trading_pair: The market to place order + :param amount: The order amount (in base token value) + :param order_type: The order type + :param price: The order price + """ + if not order_type.is_limit_type(): + raise Exception(f"Unsupported order type: {order_type}") + trading_rule = self._trading_rules[trading_pair] + + amount = self.quantize_order_amount(trading_pair, amount) + price = self.quantize_order_price(trading_pair, price) + if amount < trading_rule.min_order_size: + raise ValueError(f"Buy order amount {amount} is lower than the minimum order size " + f"{trading_rule.min_order_size}.") + api_params = {"instrument_name": hitbtc_utils.convert_to_exchange_trading_pair(trading_pair), + "side": trade_type.name, + "type": "LIMIT", + "price": f"{price:f}", + "quantity": f"{amount:f}", + "client_oid": order_id + } + if order_type is OrderType.LIMIT_MAKER: + api_params["exec_inst"] = "POST_ONLY" + self.start_tracking_order(order_id, + None, + trading_pair, + trade_type, + price, + amount, + order_type + ) + try: + order_result = await self._api_request("post", "private/create-order", api_params, True) + exchange_order_id = str(order_result["result"]["order_id"]) + tracked_order = self._in_flight_orders.get(order_id) + if tracked_order is not None: + self.logger().info(f"Created {order_type.name} {trade_type.name} order {order_id} for " + f"{amount} {trading_pair}.") + tracked_order.update_exchange_order_id(exchange_order_id) + + event_tag = MarketEvent.BuyOrderCreated if trade_type is TradeType.BUY else MarketEvent.SellOrderCreated + event_class = BuyOrderCreatedEvent if trade_type is TradeType.BUY else SellOrderCreatedEvent + self.trigger_event(event_tag, + event_class( + self.current_timestamp, + order_type, + trading_pair, + amount, + price, + order_id + )) + except asyncio.CancelledError: + raise + except Exception as e: + self.stop_tracking_order(order_id) + self.logger().network( + f"Error submitting {trade_type.name} {order_type.name} order to HitBTC for " + f"{amount} {trading_pair} " + f"{price}.", + exc_info=True, + app_warning_msg=str(e) + ) + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent(self.current_timestamp, order_id, order_type)) + + def start_tracking_order(self, + order_id: str, + exchange_order_id: str, + trading_pair: str, + trade_type: TradeType, + price: Decimal, + amount: Decimal, + order_type: OrderType): + """ + Starts tracking an order by simply adding it into _in_flight_orders dictionary. + """ + self._in_flight_orders[order_id] = HitBTCInFlightOrder( + client_order_id=order_id, + exchange_order_id=exchange_order_id, + trading_pair=trading_pair, + order_type=order_type, + trade_type=trade_type, + price=price, + amount=amount + ) + + def stop_tracking_order(self, order_id: str): + """ + Stops tracking an order by simply removing it from _in_flight_orders dictionary. + """ + if order_id in self._in_flight_orders: + del self._in_flight_orders[order_id] + + async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: + """ + Executes order cancellation process by first calling cancel-order API. The API result doesn't confirm whether + the cancellation is successful, it simply states it receives the request. + :param trading_pair: The market trading pair + :param order_id: The internal order id + order.last_state to change to CANCELED + """ + try: + tracked_order = self._in_flight_orders.get(order_id) + if tracked_order is None: + raise ValueError(f"Failed to cancel order - {order_id}. Order not found.") + if tracked_order.exchange_order_id is None: + await tracked_order.get_exchange_order_id() + ex_order_id = tracked_order.exchange_order_id + await self._api_request( + "post", + "private/cancel-order", + {"instrument_name": hitbtc_utils.convert_to_exchange_trading_pair(trading_pair), + "order_id": ex_order_id}, + True + ) + return order_id + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().network( + f"Failed to cancel order {order_id}: {str(e)}", + exc_info=True, + app_warning_msg=f"Failed to cancel the order {order_id} on HitBTC. " + f"Check API key and network connection." + ) + + async def _status_polling_loop(self): + """ + Periodically update user balances and order status via REST API. This serves as a fallback measure for web + socket API updates. + """ + while True: + try: + self._poll_notifier = asyncio.Event() + await self._poll_notifier.wait() + await safe_gather( + self._update_balances(), + self._update_order_status(), + ) + self._last_poll_timestamp = self.current_timestamp + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().error(str(e), exc_info=True) + self.logger().network("Unexpected error while fetching account updates.", + exc_info=True, + app_warning_msg="Could not fetch account updates from HitBTC. " + "Check API key and network connection.") + await asyncio.sleep(0.5) + + async def _update_balances(self): + """ + Calls REST API to update total and available balances. + """ + local_asset_names = set(self._account_balances.keys()) + remote_asset_names = set() + account_info = await self._api_request("post", "private/get-account-summary", {}, True) + for account in account_info["result"]["accounts"]: + asset_name = account["currency"] + self._account_available_balances[asset_name] = Decimal(str(account["available"])) + self._account_balances[asset_name] = Decimal(str(account["balance"])) + remote_asset_names.add(asset_name) + + asset_names_to_remove = local_asset_names.difference(remote_asset_names) + for asset_name in asset_names_to_remove: + del self._account_available_balances[asset_name] + del self._account_balances[asset_name] + + async def _update_order_status(self): + """ + Calls REST API to get status update for each in-flight order. + """ + last_tick = int(self._last_poll_timestamp / self.UPDATE_ORDER_STATUS_MIN_INTERVAL) + current_tick = int(self.current_timestamp / self.UPDATE_ORDER_STATUS_MIN_INTERVAL) + + if current_tick > last_tick and len(self._in_flight_orders) > 0: + tracked_orders = list(self._in_flight_orders.values()) + tasks = [] + for tracked_order in tracked_orders: + order_id = await tracked_order.get_exchange_order_id() + tasks.append(self._api_request("post", + "private/get-order-detail", + {"order_id": order_id}, + True)) + self.logger().debug(f"Polling for order status updates of {len(tasks)} orders.") + responses = await safe_gather(*tasks, return_exceptions=True) + for response in responses: + if isinstance(response, Exception): + raise response + if "result" not in response: + self.logger().info(f"_update_order_status result not in resp: {response}") + continue + result = response["result"] + if "trade_list" in result: + for trade_msg in result["trade_list"]: + await self._process_trade_message(trade_msg) + self._process_order_message(result["order_info"]) + + def _process_order_message(self, order_msg: Dict[str, Any]): + """ + Updates in-flight order and triggers cancellation or failure event if needed. + :param order_msg: The order response from either REST or web socket API (they are of the same format) + """ + client_order_id = order_msg["client_oid"] + if client_order_id not in self._in_flight_orders: + return + tracked_order = self._in_flight_orders[client_order_id] + # Update order execution status + tracked_order.last_state = order_msg["status"] + if tracked_order.is_cancelled: + self.logger().info(f"Successfully cancelled order {client_order_id}.") + self.trigger_event(MarketEvent.OrderCancelled, + OrderCancelledEvent( + self.current_timestamp, + client_order_id)) + tracked_order.cancelled_event.set() + self.stop_tracking_order(client_order_id) + elif tracked_order.is_failure: + self.logger().info(f"The market order {client_order_id} has failed according to order status API. " + f"Reason: {hitbtc_utils.get_api_reason(order_msg['reason'])}") + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent( + self.current_timestamp, + client_order_id, + tracked_order.order_type + )) + self.stop_tracking_order(client_order_id) + + async def _process_trade_message(self, trade_msg: Dict[str, Any]): + """ + Updates in-flight order and trigger order filled event for trade message received. Triggers order completed + event if the total executed amount equals to the specified order amount. + """ + for order in self._in_flight_orders.values(): + await order.get_exchange_order_id() + track_order = [o for o in self._in_flight_orders.values() if trade_msg["order_id"] == o.exchange_order_id] + if not track_order: + return + tracked_order = track_order[0] + updated = tracked_order.update_with_trade_update(trade_msg) + if not updated: + return + self.trigger_event( + MarketEvent.OrderFilled, + OrderFilledEvent( + self.current_timestamp, + tracked_order.client_order_id, + tracked_order.trading_pair, + tracked_order.trade_type, + tracked_order.order_type, + Decimal(str(trade_msg["traded_price"])), + Decimal(str(trade_msg["traded_quantity"])), + TradeFee(0.0, [(trade_msg["fee_currency"], Decimal(str(trade_msg["fee"])))]), + exchange_trade_id=trade_msg["order_id"] + ) + ) + if math.isclose(tracked_order.executed_amount_base, tracked_order.amount) or \ + tracked_order.executed_amount_base >= tracked_order.amount: + tracked_order.last_state = "FILLED" + self.logger().info(f"The {tracked_order.trade_type.name} order " + f"{tracked_order.client_order_id} has completed " + f"according to order status API.") + event_tag = MarketEvent.BuyOrderCompleted if tracked_order.trade_type is TradeType.BUY \ + else MarketEvent.SellOrderCompleted + event_class = BuyOrderCompletedEvent if tracked_order.trade_type is TradeType.BUY \ + else SellOrderCompletedEvent + self.trigger_event(event_tag, + event_class(self.current_timestamp, + tracked_order.client_order_id, + tracked_order.base_asset, + tracked_order.quote_asset, + tracked_order.fee_asset, + tracked_order.executed_amount_base, + tracked_order.executed_amount_quote, + tracked_order.fee_paid, + tracked_order.order_type)) + self.stop_tracking_order(tracked_order.client_order_id) + + async def cancel_all(self, timeout_seconds: float): + """ + Cancels all in-flight orders and waits for cancellation results. + Used by bot's top level stop and exit commands (cancelling outstanding orders on exit) + :param timeout_seconds: The timeout at which the operation will be canceled. + :returns List of CancellationResult which indicates whether each order is successfully cancelled. + """ + if self._trading_pairs is None: + raise Exception("cancel_all can only be used when trading_pairs are specified.") + cancellation_results = [] + try: + for trading_pair in self._trading_pairs: + await self._api_request( + "post", + "private/cancel-all-orders", + {"instrument_name": hitbtc_utils.convert_to_exchange_trading_pair(trading_pair)}, + True + ) + open_orders = await self.get_open_orders() + for cl_order_id, tracked_order in self._in_flight_orders.items(): + open_order = [o for o in open_orders if o.client_order_id == cl_order_id] + if not open_order: + cancellation_results.append(CancellationResult(cl_order_id, True)) + self.trigger_event(MarketEvent.OrderCancelled, + OrderCancelledEvent(self.current_timestamp, cl_order_id)) + else: + cancellation_results.append(CancellationResult(cl_order_id, False)) + except Exception: + self.logger().network( + "Failed to cancel all orders.", + exc_info=True, + app_warning_msg="Failed to cancel all orders on HitBTC. Check API key and network connection." + ) + return cancellation_results + + def tick(self, timestamp: float): + """ + Is called automatically by the clock for each clock's tick (1 second by default). + It checks if status polling task is due for execution. + """ + now = time.time() + poll_interval = (self.SHORT_POLL_INTERVAL + if now - self._user_stream_tracker.last_recv_time > 60.0 + else self.LONG_POLL_INTERVAL) + last_tick = int(self._last_timestamp / poll_interval) + current_tick = int(timestamp / poll_interval) + if current_tick > last_tick: + if not self._poll_notifier.is_set(): + self._poll_notifier.set() + self._last_timestamp = timestamp + + def get_fee(self, + base_currency: str, + quote_currency: str, + order_type: OrderType, + order_side: TradeType, + amount: Decimal, + price: Decimal = s_decimal_NaN) -> TradeFee: + """ + To get trading fee, this function is simplified by using fee override configuration. Most parameters to this + function are ignore except order_type. Use OrderType.LIMIT_MAKER to specify you want trading fee for + maker order. + """ + is_maker = order_type is OrderType.LIMIT_MAKER + return TradeFee(percent=self.estimate_fee_pct(is_maker)) + + async def _iter_user_event_queue(self) -> AsyncIterable[Dict[str, any]]: + while True: + try: + yield await self._user_stream_tracker.user_stream.get() + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unknown error. Retrying after 1 seconds.", + exc_info=True, + app_warning_msg="Could not fetch user events from HitBTC. Check API key and network connection." + ) + await asyncio.sleep(1.0) + + async def _user_stream_event_listener(self): + """ + Listens to message in _user_stream_tracker.user_stream queue. The messages are put in by + HitBTCAPIUserStreamDataSource. + """ + async for event_message in self._iter_user_event_queue(): + try: + if "result" not in event_message or "channel" not in event_message["result"]: + continue + channel = event_message["result"]["channel"] + if "user.trade" in channel: + for trade_msg in event_message["result"]["data"]: + await self._process_trade_message(trade_msg) + elif "user.order" in channel: + for order_msg in event_message["result"]["data"]: + self._process_order_message(order_msg) + elif channel == "user.balance": + balances = event_message["result"]["data"] + for balance_entry in balances: + asset_name = balance_entry["currency"] + self._account_balances[asset_name] = Decimal(str(balance_entry["balance"])) + self._account_available_balances[asset_name] = Decimal(str(balance_entry["available"])) + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error in user stream listener loop.", exc_info=True) + await asyncio.sleep(5.0) + + async def get_open_orders(self) -> List[OpenOrder]: + result = await self._api_request( + "post", + "private/get-open-orders", + {}, + True + ) + ret_val = [] + for order in result["result"]["order_list"]: + if hitbtc_utils.HBOT_BROKER_ID not in order["client_oid"]: + continue + if order["type"] != "LIMIT": + raise Exception(f"Unsupported order type {order['type']}") + ret_val.append( + OpenOrder( + client_order_id=order["client_oid"], + trading_pair=hitbtc_utils.convert_from_exchange_trading_pair(order["instrument_name"]), + price=Decimal(str(order["price"])), + amount=Decimal(str(order["quantity"])), + executed_amount=Decimal(str(order["cumulative_quantity"])), + status=order["status"], + order_type=OrderType.LIMIT, + is_buy=True if order["side"].lower() == "buy" else False, + time=int(order["create_time"]), + exchange_order_id=order["order_id"] + ) + ) + return ret_val diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py b/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py new file mode 100644 index 0000000000..d51c40ebce --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py @@ -0,0 +1,99 @@ +from decimal import Decimal +from typing import ( + Any, + Dict, + Optional, +) +import asyncio +from hummingbot.core.event.events import ( + OrderType, + TradeType +) +from hummingbot.connector.in_flight_order_base import InFlightOrderBase + + +class HitBTCInFlightOrder(InFlightOrderBase): + def __init__(self, + client_order_id: str, + exchange_order_id: Optional[str], + trading_pair: str, + order_type: OrderType, + trade_type: TradeType, + price: Decimal, + amount: Decimal, + initial_state: str = "OPEN"): + super().__init__( + client_order_id, + exchange_order_id, + trading_pair, + order_type, + trade_type, + price, + amount, + initial_state, + ) + self.trade_id_set = set() + self.cancelled_event = asyncio.Event() + + @property + def is_done(self) -> bool: + return self.last_state in {"FILLED", "CANCELED", "REJECTED", "EXPIRED"} + + @property + def is_failure(self) -> bool: + return self.last_state in {"REJECTED"} + + @property + def is_cancelled(self) -> bool: + return self.last_state in {"CANCELED", "EXPIRED"} + + # @property + # def order_type_description(self) -> str: + # """ + # :return: Order description string . One of ["limit buy" / "limit sell" / "market buy" / "market sell"] + # """ + # order_type = "market" if self.order_type is OrderType.MARKET else "limit" + # side = "buy" if self.trade_type == TradeType.BUY else "sell" + # return f"{order_type} {side}" + + @classmethod + def from_json(cls, data: Dict[str, Any]) -> InFlightOrderBase: + """ + :param data: json data from API + :return: formatted InFlightOrder + """ + retval = HitBTCInFlightOrder( + data["client_order_id"], + data["exchange_order_id"], + data["trading_pair"], + getattr(OrderType, data["order_type"]), + getattr(TradeType, data["trade_type"]), + Decimal(data["price"]), + Decimal(data["amount"]), + data["last_state"] + ) + retval.executed_amount_base = Decimal(data["executed_amount_base"]) + retval.executed_amount_quote = Decimal(data["executed_amount_quote"]) + retval.fee_asset = data["fee_asset"] + retval.fee_paid = Decimal(data["fee_paid"]) + retval.last_state = data["last_state"] + return retval + + def update_with_trade_update(self, trade_update: Dict[str, Any]) -> bool: + """ + Updates the in flight order with trade update (from private/get-order-detail end point) + return: True if the order gets updated otherwise False + """ + trade_id = trade_update["trade_id"] + # trade_update["orderId"] is type int + if str(trade_update["order_id"]) != self.exchange_order_id or trade_id in self.trade_id_set: + # trade already recorded + return False + self.trade_id_set.add(trade_id) + self.executed_amount_base += Decimal(str(trade_update["traded_quantity"])) + self.fee_paid += Decimal(str(trade_update["fee"])) + self.executed_amount_quote += (Decimal(str(trade_update["traded_price"])) * + Decimal(str(trade_update["traded_quantity"]))) + if not self.fee_asset: + self.fee_asset = trade_update["fee_currency"] + return True diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py new file mode 100644 index 0000000000..d71fbca20d --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python + +import logging +import hummingbot.connector.exchange.hitbtc.hitbtc_constants as constants + +from sqlalchemy.engine import RowProxy +from typing import ( + Optional, + Dict, + List, Any) +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_message import ( + OrderBookMessage, OrderBookMessageType +) +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_message import HitBTCOrderBookMessage + +_logger = None + + +class HitBTCOrderBook(OrderBook): + @classmethod + def logger(cls) -> HummingbotLogger: + global _logger + if _logger is None: + _logger = logging.getLogger(__name__) + return _logger + + @classmethod + def snapshot_message_from_exchange(cls, + msg: Dict[str, any], + timestamp: float, + metadata: Optional[Dict] = None): + """ + Convert json snapshot data into standard OrderBookMessage format + :param msg: json snapshot data from live web socket stream + :param timestamp: timestamp attached to incoming data + :return: HitBTCOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + return HitBTCOrderBookMessage( + message_type=OrderBookMessageType.SNAPSHOT, + content=msg, + timestamp=timestamp + ) + + @classmethod + def snapshot_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of snapshot data into standard OrderBookMessage format + :param record: a row of snapshot data from the database + :return: HitBTCOrderBookMessage + """ + return HitBTCOrderBookMessage( + message_type=OrderBookMessageType.SNAPSHOT, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def diff_message_from_exchange(cls, + msg: Dict[str, any], + timestamp: Optional[float] = None, + metadata: Optional[Dict] = None): + """ + Convert json diff data into standard OrderBookMessage format + :param msg: json diff data from live web socket stream + :param timestamp: timestamp attached to incoming data + :return: HitBTCOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + return HitBTCOrderBookMessage( + message_type=OrderBookMessageType.DIFF, + content=msg, + timestamp=timestamp + ) + + @classmethod + def diff_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of diff data into standard OrderBookMessage format + :param record: a row of diff data from the database + :return: HitBTCOrderBookMessage + """ + return HitBTCOrderBookMessage( + message_type=OrderBookMessageType.DIFF, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def trade_message_from_exchange(cls, + msg: Dict[str, Any], + timestamp: Optional[float] = None, + metadata: Optional[Dict] = None): + """ + Convert a trade data into standard OrderBookMessage format + :param record: a trade data from the database + :return: HitBTCOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + msg.update({ + "exchange_order_id": msg.get("d"), + "trade_type": msg.get("s"), + "price": msg.get("p"), + "amount": msg.get("q"), + }) + + return HitBTCOrderBookMessage( + message_type=OrderBookMessageType.TRADE, + content=msg, + timestamp=timestamp + ) + + @classmethod + def trade_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of trade data into standard OrderBookMessage format + :param record: a row of trade data from the database + :return: HitBTCOrderBookMessage + """ + return HitBTCOrderBookMessage( + message_type=OrderBookMessageType.TRADE, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def from_snapshot(cls, snapshot: OrderBookMessage): + raise NotImplementedError(constants.EXCHANGE_NAME + " order book needs to retain individual order data.") + + @classmethod + def restore_from_snapshot_and_diffs(self, snapshot: OrderBookMessage, diffs: List[OrderBookMessage]): + raise NotImplementedError(constants.EXCHANGE_NAME + " order book needs to retain individual order data.") diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py new file mode 100644 index 0000000000..2f4de4e3b5 --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python + +from typing import ( + Dict, + List, + Optional, +) + +from hummingbot.core.data_type.order_book_row import OrderBookRow +from hummingbot.core.data_type.order_book_message import ( + OrderBookMessage, + OrderBookMessageType, +) + + +class HitBTCOrderBookMessage(OrderBookMessage): + def __new__( + cls, + message_type: OrderBookMessageType, + content: Dict[str, any], + timestamp: Optional[float] = None, + *args, + **kwargs, + ): + if timestamp is None: + if message_type is OrderBookMessageType.SNAPSHOT: + raise ValueError("timestamp must not be None when initializing snapshot messages.") + timestamp = content["timestamp"] + + return super(HitBTCOrderBookMessage, cls).__new__( + cls, message_type, content, timestamp=timestamp, *args, **kwargs + ) + + @property + def update_id(self) -> int: + if self.type in [OrderBookMessageType.DIFF, OrderBookMessageType.SNAPSHOT]: + return int(self.timestamp * 1e3) + else: + return -1 + + @property + def trade_id(self) -> int: + if self.type is OrderBookMessageType.TRADE: + return int(self.timestamp * 1e3) + return -1 + + @property + def trading_pair(self) -> str: + if "trading_pair" in self.content: + return self.content["trading_pair"] + elif "instrument_name" in self.content: + return self.content["instrument_name"] + + @property + def asks(self) -> List[OrderBookRow]: + asks = map(self.content["asks"], lambda ask: {"price": ask[0], "amount": ask[1]}) + + return [ + OrderBookRow(float(price), float(amount), self.update_id) for price, amount in asks + ] + + @property + def bids(self) -> List[OrderBookRow]: + bids = map(self.content["bids"], lambda bid: {"price": bid[0], "amount": bid[1]}) + + return [ + OrderBookRow(float(price), float(amount), self.update_id) for price, amount in bids + ] + + def __eq__(self, other) -> bool: + return self.type == other.type and self.timestamp == other.timestamp + + def __lt__(self, other) -> bool: + if self.timestamp != other.timestamp: + return self.timestamp < other.timestamp + else: + """ + If timestamp is the same, the ordering is snapshot < diff < trade + """ + return self.type.value < other.type.value diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py new file mode 100644 index 0000000000..39f5850bfa --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +import asyncio +import bisect +import logging +import hummingbot.connector.exchange.hitbtc.hitbtc_constants as constants +import time + +from collections import defaultdict, deque +from typing import Optional, Dict, List, Deque +from hummingbot.core.data_type.order_book_message import OrderBookMessageType +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.order_book_tracker import OrderBookTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_message import HitBTCOrderBookMessage +from hummingbot.connector.exchange.hitbtc.hitbtc_active_order_tracker import HitBTCActiveOrderTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_api_order_book_data_source import HitBTCAPIOrderBookDataSource +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book import HitBTCOrderBook + + +class HitBTCOrderBookTracker(OrderBookTracker): + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, trading_pairs: Optional[List[str]] = None,): + super().__init__(HitBTCAPIOrderBookDataSource(trading_pairs), trading_pairs) + + self._ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + self._order_book_snapshot_stream: asyncio.Queue = asyncio.Queue() + self._order_book_diff_stream: asyncio.Queue = asyncio.Queue() + self._order_book_trade_stream: asyncio.Queue = asyncio.Queue() + self._process_msg_deque_task: Optional[asyncio.Task] = None + self._past_diffs_windows: Dict[str, Deque] = {} + self._order_books: Dict[str, HitBTCOrderBook] = {} + self._saved_message_queues: Dict[str, Deque[HitBTCOrderBookMessage]] = \ + defaultdict(lambda: deque(maxlen=1000)) + self._active_order_trackers: Dict[str, HitBTCActiveOrderTracker] = defaultdict(HitBTCActiveOrderTracker) + self._order_book_stream_listener_task: Optional[asyncio.Task] = None + self._order_book_trade_listener_task: Optional[asyncio.Task] = None + + @property + def exchange_name(self) -> str: + """ + Name of the current exchange + """ + return constants.EXCHANGE_NAME + + async def _track_single_book(self, trading_pair: str): + """ + Update an order book with changes from the latest batch of received messages + """ + past_diffs_window: Deque[HitBTCOrderBookMessage] = deque() + self._past_diffs_windows[trading_pair] = past_diffs_window + + message_queue: asyncio.Queue = self._tracking_message_queues[trading_pair] + order_book: HitBTCOrderBook = self._order_books[trading_pair] + active_order_tracker: HitBTCActiveOrderTracker = self._active_order_trackers[trading_pair] + + last_message_timestamp: float = time.time() + diff_messages_accepted: int = 0 + + while True: + try: + message: HitBTCOrderBookMessage = None + saved_messages: Deque[HitBTCOrderBookMessage] = self._saved_message_queues[trading_pair] + # Process saved messages first if there are any + if len(saved_messages) > 0: + message = saved_messages.popleft() + else: + message = await message_queue.get() + + if message.type is OrderBookMessageType.DIFF: + bids, asks = active_order_tracker.convert_diff_message_to_order_book_row(message) + order_book.apply_diffs(bids, asks, message.update_id) + past_diffs_window.append(message) + while len(past_diffs_window) > self.PAST_DIFF_WINDOW_SIZE: + past_diffs_window.popleft() + diff_messages_accepted += 1 + + # Output some statistics periodically. + now: float = time.time() + if int(now / 60.0) > int(last_message_timestamp / 60.0): + self.logger().debug("Processed %d order book diffs for %s.", + diff_messages_accepted, trading_pair) + diff_messages_accepted = 0 + last_message_timestamp = now + elif message.type is OrderBookMessageType.SNAPSHOT: + past_diffs: List[HitBTCOrderBookMessage] = list(past_diffs_window) + # only replay diffs later than snapshot, first update active order with snapshot then replay diffs + replay_position = bisect.bisect_right(past_diffs, message) + replay_diffs = past_diffs[replay_position:] + s_bids, s_asks = active_order_tracker.convert_snapshot_message_to_order_book_row(message) + order_book.apply_snapshot(s_bids, s_asks, message.update_id) + for diff_message in replay_diffs: + d_bids, d_asks = active_order_tracker.convert_diff_message_to_order_book_row(diff_message) + order_book.apply_diffs(d_bids, d_asks, diff_message.update_id) + + self.logger().debug("Processed order book snapshot for %s.", trading_pair) + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + f"Unexpected error processing order book messages for {trading_pair}.", + exc_info=True, + app_warning_msg="Unexpected error processing order book messages. Retrying after 5 seconds." + ) + await asyncio.sleep(5.0) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker_entry.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker_entry.py new file mode 100644 index 0000000000..d463f3549c --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker_entry.py @@ -0,0 +1,21 @@ +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_tracker_entry import OrderBookTrackerEntry +from hummingbot.connector.exchange.hitbtc.hitbtc_active_order_tracker import HitBTCActiveOrderTracker + + +class HitBTCOrderBookTrackerEntry(OrderBookTrackerEntry): + def __init__( + self, trading_pair: str, timestamp: float, order_book: OrderBook, active_order_tracker: HitBTCActiveOrderTracker + ): + self._active_order_tracker = active_order_tracker + super(HitBTCOrderBookTrackerEntry, self).__init__(trading_pair, timestamp, order_book) + + def __repr__(self) -> str: + return ( + f"HitBTCOrderBookTrackerEntry(trading_pair='{self._trading_pair}', timestamp='{self._timestamp}', " + f"order_book='{self._order_book}')" + ) + + @property + def active_order_tracker(self) -> HitBTCActiveOrderTracker: + return self._active_order_tracker diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_user_stream_tracker.py b/hummingbot/connector/exchange/hitbtc/hitbtc_user_stream_tracker.py new file mode 100644 index 0000000000..a3b54eb2fa --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_user_stream_tracker.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +import asyncio +import logging +from typing import ( + Optional, + List, +) +from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.user_stream_tracker import ( + UserStreamTracker +) +from hummingbot.core.utils.async_utils import ( + safe_ensure_future, + safe_gather, +) +from hummingbot.connector.exchange.hitbtc.hitbtc_api_user_stream_data_source import \ + HitBTCAPIUserStreamDataSource +from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitBTCAuth +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import EXCHANGE_NAME + + +class HitBTCUserStreamTracker(UserStreamTracker): + _cbpust_logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._bust_logger is None: + cls._bust_logger = logging.getLogger(__name__) + return cls._bust_logger + + def __init__(self, + hitbtc_auth: Optional[HitBTCAuth] = None, + trading_pairs: Optional[List[str]] = []): + super().__init__() + self._hitbtc_auth: HitBTCAuth = hitbtc_auth + self._trading_pairs: List[str] = trading_pairs + self._ev_loop: asyncio.events.AbstractEventLoop = asyncio.get_event_loop() + self._data_source: Optional[UserStreamTrackerDataSource] = None + self._user_stream_tracking_task: Optional[asyncio.Task] = None + + @property + def data_source(self) -> UserStreamTrackerDataSource: + """ + *required + Initializes a user stream data source (user specific order diffs from live socket stream) + :return: OrderBookTrackerDataSource + """ + if not self._data_source: + self._data_source = HitBTCAPIUserStreamDataSource( + hitbtc_auth=self._hitbtc_auth, + trading_pairs=self._trading_pairs + ) + return self._data_source + + @property + def exchange_name(self) -> str: + """ + *required + Name of the current exchange + """ + return EXCHANGE_NAME + + async def start(self): + """ + *required + Start all listeners and tasks + """ + self._user_stream_tracking_task = safe_ensure_future( + self.data_source.listen_for_user_stream(self._ev_loop, self._user_stream) + ) + await safe_gather(self._user_stream_tracking_task) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py new file mode 100644 index 0000000000..9aaa477a61 --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py @@ -0,0 +1,90 @@ +import math +from typing import Dict, List + +from hummingbot.core.utils.tracking_nonce import get_tracking_nonce, get_tracking_nonce_low_res +from . import hitbtc_constants as Constants + +from hummingbot.client.config.config_var import ConfigVar +from hummingbot.client.config.config_methods import using_exchange + + +CENTRALIZED = True + +EXAMPLE_PAIR = "ETH-USDT" + +DEFAULT_FEES = [0.1, 0.1] + +HBOT_BROKER_ID = "HBOT-" + + +# deeply merge two dictionaries +def merge_dicts(source: Dict, destination: Dict) -> Dict: + for key, value in source.items(): + if isinstance(value, dict): + # get node or create one + node = destination.setdefault(key, {}) + merge_dicts(value, node) + else: + destination[key] = value + + return destination + + +# join paths +def join_paths(*paths: List[str]) -> str: + return "/".join(paths) + + +# get timestamp in milliseconds +def get_ms_timestamp() -> int: + return get_tracking_nonce_low_res() + + +# convert milliseconds timestamp to seconds +def ms_timestamp_to_s(ms: int) -> int: + return math.floor(ms / 1e3) + + +# Request ID class +class RequestId: + """ + Generate request ids + """ + _request_id: int = 0 + + @classmethod + def generate_request_id(cls) -> int: + return get_tracking_nonce() + + +def convert_from_exchange_trading_pair(exchange_trading_pair: str) -> str: + return exchange_trading_pair.replace("_", "-") + + +def convert_to_exchange_trading_pair(hb_trading_pair: str) -> str: + return hb_trading_pair.replace("-", "_") + + +def get_new_client_order_id(is_buy: bool, trading_pair: str) -> str: + side = "B" if is_buy else "S" + return f"{HBOT_BROKER_ID}{side}-{trading_pair}-{get_tracking_nonce()}" + + +def get_api_reason(code: str) -> str: + return Constants.API_REASONS.get(int(code), code) + + +KEYS = { + "hitbtc_api_key": + ConfigVar(key="hitbtc_api_key", + prompt="Enter your HitBTC API key >>> ", + required_if=using_exchange("hitbtc"), + is_secure=True, + is_connect_key=True), + "hitbtc_secret_key": + ConfigVar(key="hitbtc_secret_key", + prompt="Enter your HitBTC secret key >>> ", + required_if=using_exchange("hitbtc"), + is_secure=True, + is_connect_key=True), +} diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py new file mode 100644 index 0000000000..c981b02166 --- /dev/null +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +import asyncio +import copy +import logging +import websockets +import ujson +import hummingbot.connector.exchange.hitbtc.hitbtc_constants as constants +from hummingbot.core.utils.async_utils import safe_ensure_future + + +from typing import Optional, AsyncIterable, Any, List +from websockets.exceptions import ConnectionClosed +from hummingbot.logger import HummingbotLogger +from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitBTCAuth +from hummingbot.connector.exchange.hitbtc.hitbtc_utils import RequestId, get_ms_timestamp + +# reusable websocket class +# ToDo: We should eventually remove this class, and instantiate web socket connection normally (see Binance for example) + + +class HitBTCWebsocket(RequestId): + MESSAGE_TIMEOUT = 30.0 + PING_TIMEOUT = 10.0 + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, auth: Optional[HitBTCAuth] = None): + self._auth: Optional[HitBTCAuth] = auth + self._isPrivate = True if self._auth is not None else False + self._WS_URL = constants.WSS_PRIVATE_URL if self._isPrivate else constants.WSS_PUBLIC_URL + self._client: Optional[websockets.WebSocketClientProtocol] = None + + # connect to exchange + async def connect(self): + try: + self._client = await websockets.connect(self._WS_URL) + + # if auth class was passed into websocket class + # we need to emit authenticated requests + if self._isPrivate: + await self._emit("public/auth", None) + # TODO: wait for response + await asyncio.sleep(1) + + return self._client + except Exception as e: + self.logger().error(f"Websocket error: '{str(e)}'", exc_info=True) + + # disconnect from exchange + async def disconnect(self): + if self._client is None: + return + + await self._client.close() + + # receive & parse messages + async def _messages(self) -> AsyncIterable[Any]: + try: + while True: + try: + raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=self.MESSAGE_TIMEOUT) + raw_msg = ujson.loads(raw_msg_str) + if "method" in raw_msg and raw_msg["method"] == "public/heartbeat": + payload = {"id": raw_msg["id"], "method": "public/respond-heartbeat"} + safe_ensure_future(self._client.send(ujson.dumps(payload))) + yield raw_msg + except asyncio.TimeoutError: + await asyncio.wait_for(self._client.ping(), timeout=self.PING_TIMEOUT) + except asyncio.TimeoutError: + self.logger().warning("WebSocket ping timed out. Going to reconnect...") + return + except ConnectionClosed: + return + finally: + await self.disconnect() + + # emit messages + async def _emit(self, method: str, data: Optional[Any] = {}) -> int: + id = self.generate_request_id() + nonce = get_ms_timestamp() + + payload = { + "id": id, + "method": method, + "nonce": nonce, + "params": copy.deepcopy(data), + } + + if self._isPrivate: + auth = self._auth.generate_auth_dict( + method, + request_id=id, + nonce=nonce, + data=data, + ) + + payload["sig"] = auth["sig"] + payload["api_key"] = auth["api_key"] + + await self._client.send(ujson.dumps(payload)) + + return id + + # request via websocket + async def request(self, method: str, data: Optional[Any] = {}) -> int: + return await self._emit(method, data) + + # subscribe to a method + async def subscribe(self, channels: List[str]) -> int: + return await self.request("subscribe", { + "channels": channels + }) + + # unsubscribe to a method + async def unsubscribe(self, channels: List[str]) -> int: + return await self.request("unsubscribe", { + "channels": channels + }) + + # listen to messages by method + async def on_message(self) -> AsyncIterable[Any]: + async for msg in self._messages(): + yield msg From 759635e7d58f8801c7a8fe7769eeea6ae83cd199 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Mon, 1 Mar 2021 23:12:34 +0000 Subject: [PATCH 02/33] HitBTC: Update README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 10417f0950..eb9e13553d 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ We created hummingbot to promote **decentralized market-making**: enabling membe | Crypto.com | crypto_com | [Crypto.com](https://crypto.com/exchange) | 2 | [API](https://exchange-docs.crypto.com/#introduction) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | | DyDx | dydx | [dy/dx](https://dydx.exchange/) | 1 | [API](https://docs.dydx.exchange/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Eterbase | eterbase | [Eterbase](https://www.eterbase.com/) | * | [API](https://developers.eterbase.exchange/?version=latest) |![RED](https://via.placeholder.com/15/f03c15/?text=+) | +| HitBTC | hitbtc | [HitBTC](https://hitbtc.com/) | 2 | [API](https://api.hitbtc.com/) | ![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | |Huobi Global| huobi | [Huobi Global](https://www.hbg.com) | 1 | [API](https://huobiapi.github.io/docs/spot/v1/en/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | KuCoin | kucoin | [KuCoin](https://www.kucoin.com/) | 1 | [API](https://docs.kucoin.com/#general) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Kraken | kraken | [Kraken](https://www.kraken.com/) | 1 | [API](https://www.kraken.com/features/api) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | From c64cd061ba409e86fce77def428fdad7b77fe67a Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Tue, 2 Mar 2021 00:33:24 +0000 Subject: [PATCH 03/33] HitBTC: Update Constants and Utils --- .../exchange/hitbtc/hitbtc_constants.py | 87 ++++++++++------- .../connector/exchange/hitbtc/hitbtc_utils.py | 96 ++++++++++++++++++- 2 files changed, 146 insertions(+), 37 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py index fe63df019e..7530e67dfa 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py @@ -1,36 +1,59 @@ # A single source of truth for constant variables related to the exchange +class Constants: + EXCHANGE_NAME = "hitbtc" + REST_URL = "https://api.hitbtc.com/api/2" + # WSS_PRIVATE_URL = "wss://stream.crypto.com/v2/user" + WSS_PRIVATE_URL = "wss://api.hitbtc.com/api/2/ws/trading" + # WSS_PUBLIC_URL = "wss://stream.crypto.com/v2/market" + WSS_PUBLIC_URL = "wss://api.hitbtc.com/api/2/ws/public" + API_REASONS = { + 0: "Success", + 403: "Action is forbidden for account", # HTTP: 401 + 429: "Too many requests", # HTTP: 429 + 500: "Internal Server Error", # HTTP: 500 + 503: "Service Unavailable", # HTTP: 503 + 504: "Gateway Timeout", # HTTP: 504 + 1001: "Authorization required", # HTTP: 401 + 1002: "Authorization required or has been failed", # HTTP: 401 + 1003: "Action forbidden for this API key", # HTTP: 403 + 1004: "Unsupported authorization method", # HTTP: 401 + 2001: "Symbol not found", # HTTP: 400 + 2002: "Currency not found", # HTTP: 400 + 2010: "Quantity not a valid number", # HTTP: 400 + 2011: "Quantity too low", # HTTP: 400 + 2012: "Bad quantity", # HTTP: 400 + 2020: "Price not a valid number", # HTTP: 400 + 2021: "Price too low", # HTTP: 400 + 2022: "Bad price", # HTTP: 400 + 20001: "Insufficient funds", # HTTP: 400 + 20002: "Order not found", # HTTP: 400 + 20003: "Limit exceeded", # HTTP: 400 + 20004: "Transaction not found", # HTTP: 400 + 20005: "Payout not found", # HTTP: 400 + 20006: "Payout already committed", # HTTP: 400 + 20007: "Payout already rolled back", # HTTP: 400 + 20008: "Duplicate clientOrderId", # HTTP: 400 + 20009: "Price and quantity not changed", # HTTP: 400 + 20010: "Exchange temporary closed", # HTTP: 400 + 20011: "Payout address is invalid", # HTTP: 400 + 20014: "Offchain for this payout is unavailable", # HTTP: 400 + 20032: "Margin account or position not found", # HTTP: 400 + 20033: "Position not changed", # HTTP: 400 + 20034: "Position in close only state", # HTTP: 400 + 20040: "Margin trading forbidden", # HTTP: 400 + 20080: "Internal order execution deadline exceeded", # HTTP: 400. + 10001: "Validation error", # HTTP: 400 + 10021: "User disabled", # HTTP: 400 -EXCHANGE_NAME = "hitbtc" -REST_URL = "https://api.crypto.com/v2" -# WSS_PRIVATE_URL = "wss://stream.crypto.com/v2/user" -WSS_PRIVATE_URL = "wss://d289dek49b4wqs.cloudfront.net/v2/user" -# WSS_PUBLIC_URL = "wss://stream.crypto.com/v2/market" -WSS_PUBLIC_URL = "wss://d10tq1f9ygdz7y.cloudfront.net/v2/market" + } -API_REASONS = { - 0: "Success", - 10001: "Malformed request, (E.g. not using application/json for REST)", - 10002: "Not authenticated, or key/signature incorrect", - 10003: "IP address not whitelisted", - 10004: "Missing required fields", - 10005: "Disallowed based on user tier", - 10006: "Requests have exceeded rate limits", - 10007: "Nonce value differs by more than 30 seconds from server", - 10008: "Invalid method specified", - 10009: "Invalid date range", - 20001: "Duplicated record", - 20002: "Insufficient balance", - 30003: "Invalid instrument_name specified", - 30004: "Invalid side specified", - 30005: "Invalid type specified", - 30006: "Price is lower than the minimum", - 30007: "Price is higher than the maximum", - 30008: "Quantity is lower than the minimum", - 30009: "Quantity is higher than the maximum", - 30010: "Required argument is blank or missing", - 30013: "Too many decimal places for Price", - 30014: "Too many decimal places for Quantity", - 30016: "The notional amount is less than the minimum", - 30017: "The notional amount exceeds the maximum", -} + # Timeouts + MESSAGE_TIMEOUT = 30.0 + PING_TIMEOUT = 10.0 + + API_CALL_TIMEOUT = 10.0 + UPDATE_ORDERS_INTERVAL = 30.0 + + # Trading pair splitter regex + TRADING_PAIR_SPLITTER = r"^(\w+)(BTC|BCH|DAI|DDRST|EOSDT|EOS|ETH|EURS|IDRT|PAX|BUSD|GUSD|TUSD|USDC|USD)$" diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py index 9aaa477a61..c528601ba8 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py @@ -1,13 +1,25 @@ import math -from typing import Dict, List +import aiohttp +import asyncio +import random +import re +from typing import ( + Any, + Dict, + List, + Optional, + Tuple, +) from hummingbot.core.utils.tracking_nonce import get_tracking_nonce, get_tracking_nonce_low_res -from . import hitbtc_constants as Constants +from .hitbtc_constants import Constants from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.config_methods import using_exchange +TRADING_PAIR_SPLITTER = re.compile(Constants.TRADING_PAIR_SPLITTER) + CENTRALIZED = True EXAMPLE_PAIR = "ETH-USDT" @@ -17,6 +29,12 @@ HBOT_BROKER_ID = "HBOT-" +class HitBTCAPIError(IOError): + def __init__(self, error_payload: Dict[str, Any]): + super().__init__(str(error_payload)) + self.error_payload = error_payload + + # deeply merge two dictionaries def merge_dicts(source: Dict, destination: Dict) -> Dict: for key, value in source.items(): @@ -57,12 +75,26 @@ def generate_request_id(cls) -> int: return get_tracking_nonce() -def convert_from_exchange_trading_pair(exchange_trading_pair: str) -> str: - return exchange_trading_pair.replace("_", "-") +def split_trading_pair(trading_pair: str) -> Optional[Tuple[str, str]]: + try: + m = TRADING_PAIR_SPLITTER.match(trading_pair) + return m.group(1), m.group(2) + # Exceptions are now logged as warnings in trading pair fetcher + except Exception: + return None + + +def convert_from_exchange_trading_pair(ex_trading_pair: str) -> Optional[str]: + if split_trading_pair(ex_trading_pair) is None: + return None + # Altmarkets uses lowercase (btcusdt) + base_asset, quote_asset = split_trading_pair(ex_trading_pair) + return f"{base_asset.upper()}-{quote_asset.upper()}" def convert_to_exchange_trading_pair(hb_trading_pair: str) -> str: - return hb_trading_pair.replace("-", "_") + # Altmarkets uses lowercase (btcusdt) + return hb_trading_pair.replace("-", "").lower() def get_new_client_order_id(is_buy: bool, trading_pair: str) -> str: @@ -74,6 +106,60 @@ def get_api_reason(code: str) -> str: return Constants.API_REASONS.get(int(code), code) +def retry_sleep_time(try_count: int) -> float: + random.seed() + randSleep = 1 + float(random.randint(1, 10) / 100) + return float(5 + float(randSleep * (1 + (try_count ** try_count)))) + + +async def generic_api_request(method, + path_url, + params: Optional[Dict[str, Any]] = None, + client=None, + try_count: int = 0) -> Dict[str, Any]: + url = f"{Constants.REST_URL}/{path_url}" + headers = {"Content-Type": ("application/json" if method == "post" + else "application/x-www-form-urlencoded")} + http_client = client if client is not None else aiohttp.ClientSession() + response_coro = http_client.request( + method=method.upper(), url=url, headers=headers, params=params, timeout=Constants.API_CALL_TIMEOUT + ) + http_status, parsed_response, request_errors = None, None, False + try: + async with response_coro as response: + try: + parsed_response = await response.json() + except Exception: + request_errors = True + try: + parsed_response = str(await response.read()) + if len(parsed_response) > 100: + parsed_response = f"{parsed_response[:100]} ... (truncated)" + except Exception: + pass + if response.status not in [200, 201] or parsed_response is None: + request_errors = True + http_status = response.status + except Exception: + request_errors = True + if client is None: + await http_client.close() + if request_errors or parsed_response is None: + if try_count < 4: + try_count += 1 + time_sleep = retry_sleep_time(try_count) + print(f"Error fetching data from {url}. HTTP status is {http_status}. " + f"Retrying in {time_sleep:.1f}s.") + await asyncio.sleep(time_sleep) + return await generic_api_request(method=method, path_url=path_url, params=params, + client=client, try_count=try_count) + else: + print(f"Error fetching data from {url}. HTTP status is {http_status}. " + f"Final msg: {parsed_response}.") + raise HitBTCAPIError({"error": parsed_response}) + return parsed_response + + KEYS = { "hitbtc_api_key": ConfigVar(key="hitbtc_api_key", From 9d123b9c77ab2b4700a64cd9fadae8de36b6fad4 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Tue, 2 Mar 2021 03:52:45 +0000 Subject: [PATCH 04/33] HitBTC: Order Book --- .../hitbtc_api_order_book_data_source.py | 152 ++++++++++-------- .../exchange/hitbtc/hitbtc_constants.py | 15 ++ .../hitbtc/hitbtc_order_book_message.py | 4 +- .../connector/exchange/hitbtc/hitbtc_utils.py | 33 ++-- .../exchange/hitbtc/hitbtc_websocket.py | 26 +-- 5 files changed, 132 insertions(+), 98 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py index d803be91e1..87ef9e41a9 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py @@ -2,28 +2,28 @@ import asyncio import logging import time -import aiohttp import pandas as pd -import hummingbot.connector.exchange.hitbtc.hitbtc_constants as constants - +from decimal import Decimal from typing import Optional, List, Dict, Any from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.order_book_message import OrderBookMessage from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource -from hummingbot.core.utils.async_utils import safe_gather +# from hummingbot.core.utils.async_utils import safe_gather from hummingbot.logger import HummingbotLogger -from . import hitbtc_utils +from .hitbtc_constants import Constants from .hitbtc_active_order_tracker import HitBTCActiveOrderTracker from .hitbtc_order_book import HitBTCOrderBook from .hitbtc_websocket import HitBTCWebsocket -from .hitbtc_utils import ms_timestamp_to_s +from .hitbtc_utils import ( + str_date_to_ts, + convert_to_exchange_trading_pair, + convert_from_exchange_trading_pair, + generic_api_request, + HitBTCAPIError, +) class HitBTCAPIOrderBookDataSource(OrderBookTrackerDataSource): - MAX_RETRIES = 20 - MESSAGE_TIMEOUT = 30.0 - SNAPSHOT_TIMEOUT = 10.0 - _logger: Optional[HummingbotLogger] = None @classmethod @@ -38,55 +38,54 @@ def __init__(self, trading_pairs: List[str] = None): self._snapshot_msg: Dict[str, any] = {} @classmethod - async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, float]: - result = {} - async with aiohttp.ClientSession() as client: - resp = await client.get(f"{constants.REST_URL}/public/get-ticker") - resp_json = await resp.json() - for t_pair in trading_pairs: - last_trade = [o["a"] for o in resp_json["result"]["data"] if o["i"] == - hitbtc_utils.convert_to_exchange_trading_pair(t_pair)] - if last_trade and last_trade[0] is not None: - result[t_pair] = last_trade[0] - return result + async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, Decimal]: + results = {} + if len(trading_pairs) == 1: + for trading_pair in trading_pairs: + ex_pair = convert_to_exchange_trading_pair(trading_pair) + url_endpoint = Constants.ENDPOINT["TICKER_SINGLE"].format(trading_pair=ex_pair) + ticker = await generic_api_request("get", url_endpoint) + results[trading_pair] = Decimal(str(ticker["last"])) + else: + url_endpoint = Constants.ENDPOINT["TICKER"] + tickers = await generic_api_request("get", url_endpoint) + for trading_pair in trading_pairs: + ex_pair = convert_to_exchange_trading_pair(trading_pair) + ticker = list([tic for tic in tickers if tic['symbol'] == ex_pair])[0] + results[trading_pair] = Decimal(str(ticker["last"])) + return results @staticmethod async def fetch_trading_pairs() -> List[str]: - async with aiohttp.ClientSession() as client: - async with client.get(f"{constants.REST_URL}/public/get-ticker", timeout=10) as response: - if response.status == 200: - from hummingbot.connector.exchange.hitbtc.hitbtc_utils import \ - convert_from_exchange_trading_pair - try: - data: Dict[str, Any] = await response.json() - return [convert_from_exchange_trading_pair(item["i"]) for item in data["result"]["data"]] - except Exception: - pass - # Do nothing if the request fails -- there will be no autocomplete for kucoin trading pairs - return [] + try: + symbols: List[Dict[str, Any]] = await generic_api_request("get", Constants.ENDPOINT["SYMBOL"]) + return [convert_from_exchange_trading_pair(sym["id"]) for sym in symbols] + except Exception: + # Do nothing if the request fails -- there will be no autocomplete for huobi trading pairs + pass + return [] @staticmethod async def get_order_book_data(trading_pair: str) -> Dict[str, any]: """ Get whole orderbook """ - async with aiohttp.ClientSession() as client: - orderbook_response = await client.get( - f"{constants.REST_URL}/public/get-book?depth=150&instrument_name=" - f"{hitbtc_utils.convert_to_exchange_trading_pair(trading_pair)}" + try: + ex_pair = convert_to_exchange_trading_pair(trading_pair) + orderbook_response = await generic_api_request("get", + Constants.ENDPOINT["ORDER_BOOK"], + params={ + "limit": 150, + "symbols": ex_pair + }) + orderbook_data = orderbook_response[ex_pair] + return orderbook_data + except HitBTCAPIError as e: + raise IOError( + f"Error fetching OrderBook for {trading_pair} at {Constants.EXCHANGE_NAME}. " + f"HTTP status is {e.error_payload['status']}. Error is {e.error_payload['error']}." ) - if orderbook_response.status != 200: - raise IOError( - f"Error fetching OrderBook for {trading_pair} at {constants.EXCHANGE_NAME}. " - f"HTTP status is {orderbook_response.status}." - ) - - orderbook_data: List[Dict[str, Any]] = await safe_gather(orderbook_response.json()) - orderbook_data = orderbook_data[0]["result"]["data"][0] - - return orderbook_data - async def get_new_order_book(self, trading_pair: str) -> OrderBook: snapshot: Dict[str, Any] = await self.get_order_book_data(trading_pair) snapshot_timestamp: float = time.time() @@ -110,22 +109,26 @@ async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asynci ws = HitBTCWebsocket() await ws.connect() - await ws.subscribe(list(map( - lambda pair: f"trade.{hitbtc_utils.convert_to_exchange_trading_pair(pair)}", - self._trading_pairs - ))) + for pair in self._trading_pairs: + await ws.subscribe("Trades", convert_to_exchange_trading_pair(pair)) async for response in ws.on_message(): - if response.get("result") is None: + print(f"WS1: {response}") + method: str = response.get("method", None) + trades_data: str = response.get("params", None) + + if trades_data is None or method != Constants.WSS_METHODS['TRADES_UPDATE']: continue - for trade in response["result"]["data"]: + pair: str = convert_from_exchange_trading_pair(response["params"]["symbol"]) + + for trade in trades_data["data"]: trade: Dict[Any] = trade - trade_timestamp: int = ms_timestamp_to_s(trade["t"]) + trade_timestamp: int = str_date_to_ts(trade["timestamp"]) trade_msg: OrderBookMessage = HitBTCOrderBook.trade_message_from_exchange( trade, trade_timestamp, - metadata={"trading_pair": hitbtc_utils.convert_from_exchange_trading_pair(trade["i"])} + metadata={"trading_pair": pair} ) output.put_nowait(trade_msg) @@ -146,25 +149,34 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp ws = HitBTCWebsocket() await ws.connect() - await ws.subscribe(list(map( - lambda pair: f"book.{hitbtc_utils.convert_to_exchange_trading_pair(pair)}.150", - self._trading_pairs - ))) + order_book_methods = [ + Constants.WSS_METHODS['ORDER_SNAPSHOT'], + Constants.WSS_METHODS['ORDER_UPDATE'], + ] + + for pair in self._trading_pairs: + await ws.subscribe("Orderbook", convert_to_exchange_trading_pair(pair)) async for response in ws.on_message(): - if response.get("result") is None: + print(f"WS2: {response}") + + method: str = response.get("method", None) + order_book_data: str = response.get("params", None) + + if order_book_data is None or method not in order_book_methods: continue - order_book_data = response["result"]["data"][0] - timestamp: int = ms_timestamp_to_s(order_book_data["t"]) - # data in this channel is not order book diff but the entire order book (up to depth 150). - # so we need to convert it into a order book snapshot. - # HitBTC does not offer order book diff ws updates. - orderbook_msg: OrderBookMessage = HitBTCOrderBook.snapshot_message_from_exchange( + timestamp: int = str_date_to_ts(order_book_data["timestamp"]) + pair: str = convert_from_exchange_trading_pair(order_book_data["symbol"]) + + order_book_msg_cls = (HitBTCOrderBook.diff_message_from_exchange + if method == Constants.WSS_METHODS['ORDER_UPDATE'] else + HitBTCOrderBook.snapshot_message_from_exchange) + + orderbook_msg: OrderBookMessage = order_book_msg_cls( order_book_data, timestamp, - metadata={"trading_pair": hitbtc_utils.convert_from_exchange_trading_pair( - response["result"]["instrument_name"])} + metadata={"trading_pair": pair} ) output.put_nowait(orderbook_msg) @@ -190,7 +202,7 @@ async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, for trading_pair in self._trading_pairs: try: snapshot: Dict[str, any] = await self.get_order_book_data(trading_pair) - snapshot_timestamp: int = ms_timestamp_to_s(snapshot["t"]) + snapshot_timestamp: int = str_date_to_ts(snapshot["timestamp"]) snapshot_msg: OrderBookMessage = HitBTCOrderBook.snapshot_message_from_exchange( snapshot, snapshot_timestamp, diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py index 7530e67dfa..b167b1f82f 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py @@ -7,6 +7,21 @@ class Constants: # WSS_PUBLIC_URL = "wss://stream.crypto.com/v2/market" WSS_PUBLIC_URL = "wss://api.hitbtc.com/api/2/ws/public" + ENDPOINT = { + # Public Endpoints + "TICKER": "public/ticker", + "TICKER_SINGLE": "public/ticker/{trading_pair}", + "SYMBOL": "public/symbol", + "ORDER_BOOK": "public/orderbook", + } + + WSS_METHODS = { + "ORDER_SNAPSHOT": "snapshotOrderbook", + "ORDER_UPDATE": "updateOrderbook", + "TRADES_SNAPSHOT": "snapshotTrades", + "TRADES_UPDATE": "updateTrades", + } + API_REASONS = { 0: "Success", 403: "Action is forbidden for account", # HTTP: 401 diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py index 2f4de4e3b5..a54c9f1c5d 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py @@ -53,7 +53,7 @@ def trading_pair(self) -> str: @property def asks(self) -> List[OrderBookRow]: - asks = map(self.content["asks"], lambda ask: {"price": ask[0], "amount": ask[1]}) + asks = map(self.content["ask"], lambda ask: {"price": ask["price"], "size": ask["size"]}) return [ OrderBookRow(float(price), float(amount), self.update_id) for price, amount in asks @@ -61,7 +61,7 @@ def asks(self) -> List[OrderBookRow]: @property def bids(self) -> List[OrderBookRow]: - bids = map(self.content["bids"], lambda bid: {"price": bid[0], "amount": bid[1]}) + bids = map(self.content["bid"], lambda bid: {"price": bid["price"], "size": bid["size"]}) return [ OrderBookRow(float(price), float(amount), self.update_id) for price, amount in bids diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py index c528601ba8..ed6ab7feee 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py @@ -1,8 +1,8 @@ -import math import aiohttp import asyncio import random import re +from dateutil.parser import parse as dateparse from typing import ( Any, Dict, @@ -58,9 +58,9 @@ def get_ms_timestamp() -> int: return get_tracking_nonce_low_res() -# convert milliseconds timestamp to seconds -def ms_timestamp_to_s(ms: int) -> int: - return math.floor(ms / 1e3) +# convert date string to timestamp +def str_date_to_ts(date: str) -> int: + return int(dateparse(date).timestamp()) # Request ID class @@ -109,24 +109,24 @@ def get_api_reason(code: str) -> str: def retry_sleep_time(try_count: int) -> float: random.seed() randSleep = 1 + float(random.randint(1, 10) / 100) - return float(5 + float(randSleep * (1 + (try_count ** try_count)))) + return float(2 + float(randSleep * (1 + (try_count ** try_count)))) async def generic_api_request(method, - path_url, + endpoint, params: Optional[Dict[str, Any]] = None, - client=None, + shared_client=None, try_count: int = 0) -> Dict[str, Any]: - url = f"{Constants.REST_URL}/{path_url}" - headers = {"Content-Type": ("application/json" if method == "post" - else "application/x-www-form-urlencoded")} - http_client = client if client is not None else aiohttp.ClientSession() + url = f"{Constants.REST_URL}/{endpoint}" + headers = {"Content-Type": "application/json"} + http_client = shared_client if shared_client is not None else aiohttp.ClientSession() response_coro = http_client.request( method=method.upper(), url=url, headers=headers, params=params, timeout=Constants.API_CALL_TIMEOUT ) http_status, parsed_response, request_errors = None, None, False try: async with response_coro as response: + http_status = response.status try: parsed_response = await response.json() except Exception: @@ -139,24 +139,23 @@ async def generic_api_request(method, pass if response.status not in [200, 201] or parsed_response is None: request_errors = True - http_status = response.status except Exception: request_errors = True - if client is None: + if shared_client is None: await http_client.close() if request_errors or parsed_response is None: if try_count < 4: try_count += 1 time_sleep = retry_sleep_time(try_count) print(f"Error fetching data from {url}. HTTP status is {http_status}. " - f"Retrying in {time_sleep:.1f}s.") + f"Retrying in {time_sleep:.0f}s.") await asyncio.sleep(time_sleep) - return await generic_api_request(method=method, path_url=path_url, params=params, - client=client, try_count=try_count) + return await generic_api_request(method=method, endpoint=endpoint, params=params, + shared_client=shared_client, try_count=try_count) else: print(f"Error fetching data from {url}. HTTP status is {http_status}. " f"Final msg: {parsed_response}.") - raise HitBTCAPIError({"error": parsed_response}) + raise HitBTCAPIError({"error": parsed_response, "status": http_status}) return parsed_response diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py index c981b02166..ae04c2a334 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py @@ -8,7 +8,11 @@ from hummingbot.core.utils.async_utils import safe_ensure_future -from typing import Optional, AsyncIterable, Any, List +from typing import ( + Optional, + AsyncIterable, + Any, +) from websockets.exceptions import ConnectionClosed from hummingbot.logger import HummingbotLogger from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitBTCAuth @@ -111,16 +115,20 @@ async def request(self, method: str, data: Optional[Any] = {}) -> int: return await self._emit(method, data) # subscribe to a method - async def subscribe(self, channels: List[str]) -> int: - return await self.request("subscribe", { - "channels": channels - }) + async def subscribe(self, + channel: str, + trading_pair: str, + params: Optional[Any] = {}) -> int: + params['symbol'] = trading_pair + return await self.request(f"subscribe{channel}", params) # unsubscribe to a method - async def unsubscribe(self, channels: List[str]) -> int: - return await self.request("unsubscribe", { - "channels": channels - }) + async def unsubscribe(self, + channel: str, + trading_pair: str, + params: Optional[Any] = {}) -> int: + params['symbol'] = trading_pair + return await self.request(f"unsubscribe{channel}", params) # listen to messages by method async def on_message(self) -> AsyncIterable[Any]: From 4b0733c83ac7984e7f4e07333765b38e8cb4cd89 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 3 Mar 2021 02:46:13 +0000 Subject: [PATCH 05/33] HitBTC: Work on order book / WS --- .../hitbtc/hitbtc_active_order_tracker.pyx | 12 ++++++------ .../hitbtc_api_order_book_data_source.py | 19 ++++++++----------- .../exchange/hitbtc/hitbtc_websocket.py | 6 ++---- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx index a1fc15f519..6ba7d6a3be 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx @@ -46,7 +46,7 @@ cdef class HitBTCActiveOrderTracker: def get_rates_and_quantities(self, entry) -> tuple: # price, quantity - return float(entry[0]), float(entry[1]) + return float(entry["price"]), float(entry["size"]) cdef tuple c_convert_diff_message_to_np_arrays(self, object message): cdef: @@ -61,8 +61,8 @@ cdef class HitBTCActiveOrderTracker: double timestamp = message.timestamp double amount = 0 - bid_entries = content["bids"] - ask_entries = content["asks"] + bid_entries = content["bid"] + ask_entries = content["ask"] bids = s_empty_diff asks = s_empty_diff @@ -104,7 +104,7 @@ cdef class HitBTCActiveOrderTracker: timestamp = message.timestamp content = message.content - for snapshot_orders, active_orders in [(content["bids"], self._active_bids), (content["asks"], self.active_asks)]: + for snapshot_orders, active_orders in [(content["bid"], self._active_bids), (content["ask"], self.active_asks)]: for order in snapshot_orders: price, amount = self.get_rates_and_quantities(order) @@ -146,13 +146,13 @@ cdef class HitBTCActiveOrderTracker: cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message): cdef: - double trade_type_value = 2.0 + double trade_type_value = 1.0 if message.content["side"] == "buy" else 2.0 timestamp = message.timestamp content = message.content return np.array( - [timestamp, trade_type_value, float(content["price"]), float(content["size"])], + [timestamp, trade_type_value, float(content["price"]), float(content["quantity"])], dtype="float64" ) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py index 87ef9e41a9..253700146a 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py @@ -40,19 +40,16 @@ def __init__(self, trading_pairs: List[str] = None): @classmethod async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, Decimal]: results = {} - if len(trading_pairs) == 1: - for trading_pair in trading_pairs: - ex_pair = convert_to_exchange_trading_pair(trading_pair) + if len(trading_pairs) > 1: + tickers = await generic_api_request("get", Constants.ENDPOINT["TICKER"]) + for trading_pair in trading_pairs: + ex_pair = convert_to_exchange_trading_pair(trading_pair) + if len(trading_pairs) > 1: + ticker = list([tic for tic in tickers if tic['symbol'] == ex_pair])[0] + else: url_endpoint = Constants.ENDPOINT["TICKER_SINGLE"].format(trading_pair=ex_pair) ticker = await generic_api_request("get", url_endpoint) - results[trading_pair] = Decimal(str(ticker["last"])) - else: - url_endpoint = Constants.ENDPOINT["TICKER"] - tickers = await generic_api_request("get", url_endpoint) - for trading_pair in trading_pairs: - ex_pair = convert_to_exchange_trading_pair(trading_pair) - ticker = list([tic for tic in tickers if tic['symbol'] == ex_pair])[0] - results[trading_pair] = Decimal(str(ticker["last"])) + results[trading_pair] = Decimal(str(ticker["last"])) return results @staticmethod diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py index ae04c2a334..53a086fb0d 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py @@ -5,7 +5,6 @@ import websockets import ujson import hummingbot.connector.exchange.hitbtc.hitbtc_constants as constants -from hummingbot.core.utils.async_utils import safe_ensure_future from typing import ( @@ -69,9 +68,8 @@ async def _messages(self) -> AsyncIterable[Any]: try: raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=self.MESSAGE_TIMEOUT) raw_msg = ujson.loads(raw_msg_str) - if "method" in raw_msg and raw_msg["method"] == "public/heartbeat": - payload = {"id": raw_msg["id"], "method": "public/respond-heartbeat"} - safe_ensure_future(self._client.send(ujson.dumps(payload))) + # HitBTC doesn't support ping or heartbeat messages. + # Can handle them here if that changes - use `safe_ensure_future`. yield raw_msg except asyncio.TimeoutError: await asyncio.wait_for(self._client.ping(), timeout=self.PING_TIMEOUT) From aa77a38954c3d3779860ea383015cfb1424ef14e Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 3 Mar 2021 03:45:10 +0000 Subject: [PATCH 06/33] HitBTC: Tackle api_request and HS256 auth --- .../connector/exchange/hitbtc/hitbtc_auth.py | 63 ++++--- .../exchange/hitbtc/hitbtc_constants.py | 4 + .../exchange/hitbtc/hitbtc_exchange.py | 171 +++++++++--------- .../connector/exchange/hitbtc/hitbtc_utils.py | 16 +- .../exchange/hitbtc/hitbtc_websocket.py | 18 +- 5 files changed, 143 insertions(+), 129 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py b/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py index abd6be087e..8f0159aa6f 100755 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py @@ -1,5 +1,8 @@ import hmac import hashlib +import time +import ujson +from base64 import b64encode from typing import Dict, Any @@ -14,45 +17,57 @@ def __init__(self, api_key: str, secret_key: str): def generate_auth_dict( self, - path_url: str, - request_id: int, - nonce: int, - data: Dict[str, Any] = None + method: str, + url: str, + params: Dict[str, Any] = None ): """ - Generates authentication signature and return it in a dictionary along with other inputs - :return: a dictionary of request info including the request signature + Generates authentication signature and return it with the nonce used + :return: a tuple of the nonce used and the request signature """ + nonce = str(int(time.time())) + full_url = f"{url}" + body = "" + if len(params) > 0 and method.upper() == "GET": + query_string = "&".join([f"{k}={v}" for k, v in params.items()]) + full_url = f"{url}?{query_string}" + elif len(params) > 0 and method.upper() == "POST": + body = ujson.dumps(params) + payload = f"{method}{nonce}{full_url}{body}" - data = data or {} - data['method'] = path_url - data.update({'nonce': nonce, 'api_key': self.api_key, 'id': request_id}) - - data_params = data.get('params', {}) - if not data_params: - data['params'] = {} - params = ''.join( - f'{key}{data_params[key]}' - for key in sorted(data_params) - ) - - payload = f"{path_url}{data['id']}" \ - f"{self.api_key}{params}{data['nonce']}" - - data['sig'] = hmac.new( + sig = hmac.new( self.secret_key.encode('utf-8'), payload.encode('utf-8'), hashlib.sha256 ).hexdigest() + return (nonce, sig) + + def generate_auth_dict_ws(self, + nonce: int): + data = { + "algo": "HS256", + "pKey": self.api_key, + "nonce": nonce, + } + data['signature'] = hmac.new( + self.secret_key.encode('utf-8'), + str(nonce).encode('utf-8'), + hashlib.sha256 + ).hexdigest() return data - def get_headers(self) -> Dict[str, Any]: + def get_headers(self, + method, + url, + params) -> Dict[str, Any]: """ Generates authentication headers required by HitBTC :return: a dictionary of auth headers """ - + nonce, sig = self.generate_auth(method, url, params) + payload = b64encode(f"{self.api_key}:{nonce}:{sig}".encode('utf-8')).decode().strip() return { + "Authorization": f"HS256 {payload}", "Content-Type": 'application/json', } diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py index b167b1f82f..584abde14c 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py @@ -7,6 +7,8 @@ class Constants: # WSS_PUBLIC_URL = "wss://stream.crypto.com/v2/market" WSS_PUBLIC_URL = "wss://api.hitbtc.com/api/2/ws/public" + HBOT_BROKER_ID = "refzzz48" + ENDPOINT = { # Public Endpoints "TICKER": "public/ticker", @@ -70,5 +72,7 @@ class Constants: API_CALL_TIMEOUT = 10.0 UPDATE_ORDERS_INTERVAL = 30.0 + INTERVAL_TRADING_RULES = 120 + # Trading pair splitter regex TRADING_PAIR_SPLITTER = r"^(\w+)(BTC|BCH|DAI|DDRST|EOSDT|EOS|ETH|EURS|IDRT|PAX|BUSD|GUSD|TUSD|USDC|USD)$" diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index 835cf1d4a4..1157d6345d 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -8,7 +8,6 @@ ) from decimal import Decimal import asyncio -import json import aiohttp import math import time @@ -39,8 +38,15 @@ from hummingbot.connector.exchange.hitbtc.hitbtc_user_stream_tracker import HitBTCUserStreamTracker from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitBTCAuth from hummingbot.connector.exchange.hitbtc.hitbtc_in_flight_order import HitBTCInFlightOrder -from hummingbot.connector.exchange.hitbtc import hitbtc_utils -from hummingbot.connector.exchange.hitbtc import hitbtc_constants as Constants +from hummingbot.connector.exchange.hitbtc.hitbtc_utils import ( + convert_from_exchange_trading_pair, + convert_to_exchange_trading_pair, + get_new_client_order_id, + get_api_reason, + retry_sleep_time, + HitBTCAPIError, +) +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants from hummingbot.core.data_type.common import OpenOrder ctce_logger = None s_decimal_NaN = Decimal("nan") @@ -164,7 +170,7 @@ def supported_order_types(self) -> List[OrderType]: :return a list of OrderType supported by this connector. Note that Market order type is no longer required and will not be used. """ - return [OrderType.LIMIT, OrderType.LIMIT_MAKER] + return [OrderType.LIMIT] def start(self, clock: Clock, timestamp: float): """ @@ -218,8 +224,10 @@ async def check_network(self) -> NetworkStatus: the network connection. Simply ping the network (or call any light weight public API). """ try: - # since there is no ping endpoint, the lowest rate call is to get BTC-USDT ticker - await self._api_request("get", "public/get-ticker?instrument_name=BTC_USDT") + # since there is no ping endpoint, the lowest rate call is to get BTC-USD symbol + await self._api_request("get", + Constants.ENDPOINT['SYMBOL'], + params={'symbols': 'BTCUSD'}) except asyncio.CancelledError: raise except Exception: @@ -241,7 +249,7 @@ async def _trading_rules_polling_loop(self): while True: try: await self._update_trading_rules() - await asyncio.sleep(60) + await asyncio.sleep(Constants.INTERVAL_TRADING_RULES) except asyncio.CancelledError: raise except Exception as e: @@ -252,99 +260,100 @@ async def _trading_rules_polling_loop(self): await asyncio.sleep(0.5) async def _update_trading_rules(self): - instruments_info = await self._api_request("get", path_url="public/get-instruments") + symbols_info = await self._api_request("get", endpoint=Constants.ENDPOINT['SYMBOL']) self._trading_rules.clear() - self._trading_rules = self._format_trading_rules(instruments_info) + self._trading_rules = self._format_trading_rules(symbols_info) - def _format_trading_rules(self, instruments_info: Dict[str, Any]) -> Dict[str, TradingRule]: + def _format_trading_rules(self, symbols_info: Dict[str, Any]) -> Dict[str, TradingRule]: """ Converts json API response into a dictionary of trading rules. - :param instruments_info: The json API response + :param symbols_info: The json API response :return A dictionary of trading rules. Response Example: - { - "id": 11, - "method": "public/get-instruments", - "code": 0, - "result": { - "instruments": [ - { - "instrument_name": "ETH_CRO", - "quote_currency": "CRO", - "base_currency": "ETH", - "price_decimals": 2, - "quantity_decimals": 2 - }, - { - "instrument_name": "CRO_BTC", - "quote_currency": "BTC", - "base_currency": "CRO", - "price_decimals": 8, - "quantity_decimals": 2 - } - ] - } - } + [ + { + id: "BTCUSD", + baseCurrency: "BTC", + quoteCurrency: "USD", + quantityIncrement: "0.00001", + tickSize: "0.01", + takeLiquidityRate: "0.0025", + provideLiquidityRate: "0.001", + feeCurrency: "USD", + marginTrading: true, + maxInitialLeverage: "12.00" + } + ] """ result = {} - for rule in instruments_info["result"]["instruments"]: + for rule in symbols_info: try: - trading_pair = hitbtc_utils.convert_from_exchange_trading_pair(rule["instrument_name"]) - price_decimals = Decimal(str(rule["price_decimals"])) - quantity_decimals = Decimal(str(rule["quantity_decimals"])) - # E.g. a price decimal of 2 means 0.01 incremental. - price_step = Decimal("1") / Decimal(str(math.pow(10, price_decimals))) - quantity_step = Decimal("1") / Decimal(str(math.pow(10, quantity_decimals))) + trading_pair = convert_from_exchange_trading_pair(rule["id"]) + price_step = Decimal(str(rule["tickSize"])) + size_step = Decimal(str(rule["quantityIncrement"])) result[trading_pair] = TradingRule(trading_pair, - min_price_increment=price_step, - min_base_amount_increment=quantity_step) + min_order_size=size_step, + min_base_amount_increment=size_step, + min_price_increment=price_step) except Exception: self.logger().error(f"Error parsing the trading pair rule {rule}. Skipping.", exc_info=True) return result async def _api_request(self, method: str, - path_url: str, - params: Dict[str, Any] = {}, - is_auth_required: bool = False) -> Dict[str, Any]: + endpoint: str, + params: Optional[Dict[str, Any]] = None, + is_auth_required: bool = False, + try_count: int = 0) -> Dict[str, Any]: """ Sends an aiohttp request and waits for a response. :param method: The HTTP method, e.g. get or post - :param path_url: The path url or the API end point + :param endpoint: The path url or the API end point + :param params: Additional get/post parameters :param is_auth_required: Whether an authentication is required, when True the function will add encrypted signature to the request. :returns A response in json format. """ - url = f"{Constants.REST_URL}/{path_url}" - client = await self._http_client() + url = f"{Constants.REST_URL}/{endpoint}" + shared_client = await self._http_client() if is_auth_required: - request_id = hitbtc_utils.RequestId.generate_request_id() - data = {"params": params} - params = self._hitbtc_auth.generate_auth_dict(path_url, request_id, - hitbtc_utils.get_ms_timestamp(), data) - headers = self._hitbtc_auth.get_headers() + headers = self._hitbtc_auth.get_headers(method, url, params) else: headers = {"Content-Type": "application/json"} - - if method == "get": - response = await client.get(url, headers=headers) - elif method == "post": - post_json = json.dumps(params) - response = await client.post(url, data=post_json, headers=headers) - else: - raise NotImplementedError - + response_coro = shared_client.request( + method=method.upper(), url=url, headers=headers, params=params, timeout=Constants.API_CALL_TIMEOUT + ) + http_status, parsed_response, request_errors = None, None, False try: - parsed_response = json.loads(await response.text()) - except Exception as e: - raise IOError(f"Error parsing data from {url}. Error: {str(e)}") - if response.status != 200: - raise IOError(f"Error fetching data from {url}. HTTP status is {response.status}. " - f"Message: {parsed_response}") - if parsed_response["code"] != 0: - raise IOError(f"{url} API call failed, response: {parsed_response}") - # print(f"REQUEST: {method} {path_url} {params}") - # print(f"RESPONSE: {parsed_response}") + async with response_coro as response: + http_status = response.status + try: + parsed_response = await response.json() + except Exception: + request_errors = True + try: + parsed_response = str(await response.read()) + if len(parsed_response) > 100: + parsed_response = f"{parsed_response[:100]} ... (truncated)" + except Exception: + pass + if response.status not in [200, 201] or parsed_response is None: + request_errors = True + except Exception: + request_errors = True + if request_errors or parsed_response is None: + if try_count < 4: + try_count += 1 + time_sleep = retry_sleep_time(try_count) + self.logger().info(f"Error fetching data from {url}. HTTP status is {http_status}. " + f"Retrying in {time_sleep:.0f}s.") + await asyncio.sleep(time_sleep) + return await self._api_request(method=method, endpoint=endpoint, params=params, + is_auth_required=is_auth_required, try_count=try_count) + else: + self.logger().network(f"Error fetching data from {url}. HTTP status is {http_status}. " + f"Final msg: {parsed_response}.") + raise HitBTCAPIError({"error": parsed_response, "status": http_status}) return parsed_response def get_order_price_quantum(self, trading_pair: str, price: Decimal): @@ -377,7 +386,7 @@ def buy(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, :param price: The price (note: this is no longer optional) :returns A new internal order id """ - order_id: str = hitbtc_utils.get_new_client_order_id(True, trading_pair) + order_id: str = get_new_client_order_id(True, trading_pair) safe_ensure_future(self._create_order(TradeType.BUY, order_id, trading_pair, amount, order_type, price)) return order_id @@ -392,7 +401,7 @@ def sell(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, :param price: The price (note: this is no longer optional) :returns A new internal order id """ - order_id: str = hitbtc_utils.get_new_client_order_id(False, trading_pair) + order_id: str = get_new_client_order_id(False, trading_pair) safe_ensure_future(self._create_order(TradeType.SELL, order_id, trading_pair, amount, order_type, price)) return order_id @@ -431,7 +440,7 @@ async def _create_order(self, if amount < trading_rule.min_order_size: raise ValueError(f"Buy order amount {amount} is lower than the minimum order size " f"{trading_rule.min_order_size}.") - api_params = {"instrument_name": hitbtc_utils.convert_to_exchange_trading_pair(trading_pair), + api_params = {"instrument_name": convert_to_exchange_trading_pair(trading_pair), "side": trade_type.name, "type": "LIMIT", "price": f"{price:f}", @@ -528,7 +537,7 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: await self._api_request( "post", "private/cancel-order", - {"instrument_name": hitbtc_utils.convert_to_exchange_trading_pair(trading_pair), + {"instrument_name": convert_to_exchange_trading_pair(trading_pair), "order_id": ex_order_id}, True ) @@ -636,7 +645,7 @@ def _process_order_message(self, order_msg: Dict[str, Any]): self.stop_tracking_order(client_order_id) elif tracked_order.is_failure: self.logger().info(f"The market order {client_order_id} has failed according to order status API. " - f"Reason: {hitbtc_utils.get_api_reason(order_msg['reason'])}") + f"Reason: {get_api_reason(order_msg['reason'])}") self.trigger_event(MarketEvent.OrderFailure, MarketOrderFailureEvent( self.current_timestamp, @@ -710,7 +719,7 @@ async def cancel_all(self, timeout_seconds: float): await self._api_request( "post", "private/cancel-all-orders", - {"instrument_name": hitbtc_utils.convert_to_exchange_trading_pair(trading_pair)}, + {"instrument_name": convert_to_exchange_trading_pair(trading_pair)}, True ) open_orders = await self.get_open_orders() @@ -812,14 +821,14 @@ async def get_open_orders(self) -> List[OpenOrder]: ) ret_val = [] for order in result["result"]["order_list"]: - if hitbtc_utils.HBOT_BROKER_ID not in order["client_oid"]: + if Constants.HBOT_BROKER_ID not in order["client_oid"]: continue if order["type"] != "LIMIT": raise Exception(f"Unsupported order type {order['type']}") ret_val.append( OpenOrder( client_order_id=order["client_oid"], - trading_pair=hitbtc_utils.convert_from_exchange_trading_pair(order["instrument_name"]), + trading_pair=convert_from_exchange_trading_pair(order["instrument_name"]), price=Decimal(str(order["price"])), amount=Decimal(str(order["quantity"])), executed_amount=Decimal(str(order["cumulative_quantity"])), diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py index ed6ab7feee..a8ef8ade17 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py @@ -11,7 +11,7 @@ Tuple, ) -from hummingbot.core.utils.tracking_nonce import get_tracking_nonce, get_tracking_nonce_low_res +from hummingbot.core.utils.tracking_nonce import get_tracking_nonce from .hitbtc_constants import Constants from hummingbot.client.config.config_var import ConfigVar @@ -26,8 +26,6 @@ DEFAULT_FEES = [0.1, 0.1] -HBOT_BROKER_ID = "HBOT-" - class HitBTCAPIError(IOError): def __init__(self, error_payload: Dict[str, Any]): @@ -53,11 +51,6 @@ def join_paths(*paths: List[str]) -> str: return "/".join(paths) -# get timestamp in milliseconds -def get_ms_timestamp() -> int: - return get_tracking_nonce_low_res() - - # convert date string to timestamp def str_date_to_ts(date: str) -> int: return int(dateparse(date).timestamp()) @@ -99,7 +92,12 @@ def convert_to_exchange_trading_pair(hb_trading_pair: str) -> str: def get_new_client_order_id(is_buy: bool, trading_pair: str) -> str: side = "B" if is_buy else "S" - return f"{HBOT_BROKER_ID}{side}-{trading_pair}-{get_tracking_nonce()}" + symbols = trading_pair.split("-") + base = symbols[0].upper() + quote = symbols[1].upper() + base_str = f"{base[0]}{base[-1]}" + quote_str = f"{quote[0]}{quote[-1]}" + return f"{Constants.HBOT_BROKER_ID}-{side}-{base_str}{quote_str}-{get_tracking_nonce()}" def get_api_reason(code: str) -> str: diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py index 53a086fb0d..5aca798ac3 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py @@ -15,7 +15,7 @@ from websockets.exceptions import ConnectionClosed from hummingbot.logger import HummingbotLogger from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitBTCAuth -from hummingbot.connector.exchange.hitbtc.hitbtc_utils import RequestId, get_ms_timestamp +from hummingbot.connector.exchange.hitbtc.hitbtc_utils import RequestId # reusable websocket class # ToDo: We should eventually remove this class, and instantiate web socket connection normally (see Binance for example) @@ -46,7 +46,8 @@ async def connect(self): # if auth class was passed into websocket class # we need to emit authenticated requests if self._isPrivate: - await self._emit("public/auth", None) + auth_params = self._auth.generate_auth_dict_ws(self.generate_request_id()) + await self._emit("login", auth_params) # TODO: wait for response await asyncio.sleep(1) @@ -84,26 +85,13 @@ async def _messages(self) -> AsyncIterable[Any]: # emit messages async def _emit(self, method: str, data: Optional[Any] = {}) -> int: id = self.generate_request_id() - nonce = get_ms_timestamp() payload = { "id": id, "method": method, - "nonce": nonce, "params": copy.deepcopy(data), } - if self._isPrivate: - auth = self._auth.generate_auth_dict( - method, - request_id=id, - nonce=nonce, - data=data, - ) - - payload["sig"] = auth["sig"] - payload["api_key"] = auth["api_key"] - await self._client.send(ujson.dumps(payload)) return id From 818a44f018abfab34eee7c25c511d43342d336e4 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Thu, 4 Mar 2021 23:10:11 +0000 Subject: [PATCH 07/33] HitBTC: More work on order book/user tracker --- .../hitbtc_api_order_book_data_source.py | 8 +- .../hitbtc_api_user_stream_data_source.py | 4 +- .../exchange/hitbtc/hitbtc_constants.py | 22 ++-- .../exchange/hitbtc/hitbtc_exchange.py | 105 ++++++++++++------ .../exchange/hitbtc/hitbtc_in_flight_order.py | 41 +++++-- .../exchange/hitbtc/hitbtc_order_book.py | 8 +- .../hitbtc/hitbtc_order_book_message.py | 6 +- .../exchange/hitbtc/hitbtc_websocket.py | 21 ++-- 8 files changed, 136 insertions(+), 79 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py index 253700146a..c0af9a51eb 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py @@ -114,7 +114,7 @@ async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asynci method: str = response.get("method", None) trades_data: str = response.get("params", None) - if trades_data is None or method != Constants.WSS_METHODS['TRADES_UPDATE']: + if trades_data is None or method != Constants.WS_METHODS['TRADES_UPDATE']: continue pair: str = convert_from_exchange_trading_pair(response["params"]["symbol"]) @@ -147,8 +147,8 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp await ws.connect() order_book_methods = [ - Constants.WSS_METHODS['ORDER_SNAPSHOT'], - Constants.WSS_METHODS['ORDER_UPDATE'], + Constants.WS_METHODS['ORDER_SNAPSHOT'], + Constants.WS_METHODS['ORDER_UPDATE'], ] for pair in self._trading_pairs: @@ -167,7 +167,7 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp pair: str = convert_from_exchange_trading_pair(order_book_data["symbol"]) order_book_msg_cls = (HitBTCOrderBook.diff_message_from_exchange - if method == Constants.WSS_METHODS['ORDER_UPDATE'] else + if method == Constants.WS_METHODS['ORDER_UPDATE'] else HitBTCOrderBook.snapshot_message_from_exchange) orderbook_msg: OrderBookMessage = order_book_msg_cls( diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py index 0c0e441788..42e33359d8 100755 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py @@ -11,8 +11,6 @@ class HitBTCAPIUserStreamDataSource(UserStreamTrackerDataSource): - MAX_RETRIES = 20 - MESSAGE_TIMEOUT = 30.0 _logger: Optional[HummingbotLogger] = None @@ -42,7 +40,7 @@ async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: try: ws = HitBTCWebsocket(self._hitbtc_auth) await ws.connect() - await ws.subscribe(["user.order", "user.trade", "user.balance"]) + await ws.subscribe("Reports") async for msg in ws.on_message(): # print(f"WS_SOCKET: {msg}") yield msg diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py index 584abde14c..f6cdb5ad5f 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py @@ -2,10 +2,10 @@ class Constants: EXCHANGE_NAME = "hitbtc" REST_URL = "https://api.hitbtc.com/api/2" - # WSS_PRIVATE_URL = "wss://stream.crypto.com/v2/user" - WSS_PRIVATE_URL = "wss://api.hitbtc.com/api/2/ws/trading" - # WSS_PUBLIC_URL = "wss://stream.crypto.com/v2/market" - WSS_PUBLIC_URL = "wss://api.hitbtc.com/api/2/ws/public" + # WS_PRIVATE_URL = "wss://stream.crypto.com/v2/user" + WS_PRIVATE_URL = "wss://api.hitbtc.com/api/2/ws/trading" + # WS_PUBLIC_URL = "wss://stream.crypto.com/v2/market" + WS_PUBLIC_URL = "wss://api.hitbtc.com/api/2/ws/public" HBOT_BROKER_ID = "refzzz48" @@ -17,11 +17,13 @@ class Constants: "ORDER_BOOK": "public/orderbook", } - WSS_METHODS = { + WS_METHODS = { "ORDER_SNAPSHOT": "snapshotOrderbook", "ORDER_UPDATE": "updateOrderbook", "TRADES_SNAPSHOT": "snapshotTrades", "TRADES_UPDATE": "updateTrades", + "USER_ORDERS": "activeOrders", + "USER_TRADES": "report", } API_REASONS = { @@ -68,11 +70,15 @@ class Constants: # Timeouts MESSAGE_TIMEOUT = 30.0 PING_TIMEOUT = 10.0 - API_CALL_TIMEOUT = 10.0 - UPDATE_ORDERS_INTERVAL = 30.0 - INTERVAL_TRADING_RULES = 120 + # Intervals + # Only used when nothing is received from WS + SHORT_POLL_INTERVAL = 5.0 + # HitBTC poll interval can't be too long since we don't get balances via websockets + LONG_POLL_INTERVAL = 20.0 + # One minute should be fine for order status since we get these via WS + UPDATE_ORDER_STATUS_INTERVAL = 60.0 # Trading pair splitter regex TRADING_PAIR_SPLITTER = r"^(\w+)(BTC|BCH|DAI|DDRST|EOSDT|EOS|ETH|EURS|IDRT|PAX|BUSD|GUSD|TUSD|USDC|USD)$" diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index 1157d6345d..4a5dbcbb2d 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -42,7 +42,6 @@ convert_from_exchange_trading_pair, convert_to_exchange_trading_pair, get_new_client_order_id, - get_api_reason, retry_sleep_time, HitBTCAPIError, ) @@ -57,10 +56,6 @@ class HitBTCExchange(ExchangeBase): HitBTCExchange connects with HitBTC exchange and provides order book pricing, user account tracking and trading functionality. """ - API_CALL_TIMEOUT = 10.0 - SHORT_POLL_INTERVAL = 5.0 - UPDATE_ORDER_STATUS_MIN_INTERVAL = 10.0 - LONG_POLL_INTERVAL = 120.0 @classmethod def logger(cls) -> HummingbotLogger: @@ -598,8 +593,8 @@ async def _update_order_status(self): """ Calls REST API to get status update for each in-flight order. """ - last_tick = int(self._last_poll_timestamp / self.UPDATE_ORDER_STATUS_MIN_INTERVAL) - current_tick = int(self.current_timestamp / self.UPDATE_ORDER_STATUS_MIN_INTERVAL) + last_tick = int(self._last_poll_timestamp / Constants.UPDATE_ORDER_STATUS_INTERVAL) + current_tick = int(self.current_timestamp / Constants.UPDATE_ORDER_STATUS_INTERVAL) if current_tick > last_tick and len(self._in_flight_orders) > 0: tracked_orders = list(self._in_flight_orders.values()) @@ -615,26 +610,43 @@ async def _update_order_status(self): for response in responses: if isinstance(response, Exception): raise response - if "result" not in response: - self.logger().info(f"_update_order_status result not in resp: {response}") + if "clientOrderId" not in response: + self.logger().info(f"_update_order_status clientOrderId not in resp: {response}") continue - result = response["result"] - if "trade_list" in result: - for trade_msg in result["trade_list"]: - await self._process_trade_message(trade_msg) - self._process_order_message(result["order_info"]) + self._process_order_message(response) def _process_order_message(self, order_msg: Dict[str, Any]): """ Updates in-flight order and triggers cancellation or failure event if needed. :param order_msg: The order response from either REST or web socket API (they are of the same format) + Example Order: + { + "id": "4345613661", + "clientOrderId": "57d5525562c945448e3cbd559bd068c3", + "symbol": "BCCBTC", + "side": "sell", + "status": "new", + "type": "limit", + "timeInForce": "GTC", + "quantity": "0.013", + "price": "0.100000", + "cumQuantity": "0.000", + "postOnly": false, + "createdAt": "2017-10-20T12:17:12.245Z", + "updatedAt": "2017-10-20T12:17:12.245Z", + "reportType": "status" + } """ - client_order_id = order_msg["client_oid"] + client_order_id = order_msg["clientOrderId"] if client_order_id not in self._in_flight_orders: return tracked_order = self._in_flight_orders[client_order_id] # Update order execution status tracked_order.last_state = order_msg["status"] + # update order + tracked_order.executed_amount_base = Decimal(order_msg["cumQuantity"]) + tracked_order.executed_amount_quote = Decimal(order_msg["price"]) * Decimal(order_msg["cumQuantity"]) + if tracked_order.is_cancelled: self.logger().info(f"Successfully cancelled order {client_order_id}.") self.trigger_event(MarketEvent.OrderCancelled, @@ -644,8 +656,7 @@ def _process_order_message(self, order_msg: Dict[str, Any]): tracked_order.cancelled_event.set() self.stop_tracking_order(client_order_id) elif tracked_order.is_failure: - self.logger().info(f"The market order {client_order_id} has failed according to order status API. " - f"Reason: {get_api_reason(order_msg['reason'])}") + self.logger().info(f"The market order {client_order_id} has failed according to order status API. ") self.trigger_event(MarketEvent.OrderFailure, MarketOrderFailureEvent( self.current_timestamp, @@ -658,10 +669,31 @@ async def _process_trade_message(self, trade_msg: Dict[str, Any]): """ Updates in-flight order and trigger order filled event for trade message received. Triggers order completed event if the total executed amount equals to the specified order amount. + Example Trade: + { + "id": "4345697765", + "clientOrderId": "53b7cf917963464a811a4af426102c19", + "symbol": "ETHBTC", + "side": "sell", + "status": "filled", + "type": "limit", + "timeInForce": "GTC", + "quantity": "0.001", + "price": "0.053868", + "cumQuantity": "0.001", + "postOnly": false, + "createdAt": "2017-10-20T12:20:05.952Z", + "updatedAt": "2017-10-20T12:20:38.708Z", + "reportType": "trade", + "tradeQuantity": "0.001", + "tradePrice": "0.053868", + "tradeId": 55051694, + "tradeFee": "-0.000000005" + } """ for order in self._in_flight_orders.values(): await order.get_exchange_order_id() - track_order = [o for o in self._in_flight_orders.values() if trade_msg["order_id"] == o.exchange_order_id] + track_order = [o for o in self._in_flight_orders.values() if trade_msg["id"] == o.exchange_order_id] if not track_order: return tracked_order = track_order[0] @@ -676,10 +708,10 @@ async def _process_trade_message(self, trade_msg: Dict[str, Any]): tracked_order.trading_pair, tracked_order.trade_type, tracked_order.order_type, - Decimal(str(trade_msg["traded_price"])), - Decimal(str(trade_msg["traded_quantity"])), - TradeFee(0.0, [(trade_msg["fee_currency"], Decimal(str(trade_msg["fee"])))]), - exchange_trade_id=trade_msg["order_id"] + Decimal(str(trade_msg["tradePrice"])), + Decimal(str(trade_msg["tradeQuantity"])), + TradeFee(0.0, [(tracked_order.quote_asset, -Decimal(str(trade_msg["tradeFee"])))]), + exchange_trade_id=trade_msg["id"] ) ) if math.isclose(tracked_order.executed_amount_base, tracked_order.amount) or \ @@ -745,9 +777,9 @@ def tick(self, timestamp: float): It checks if status polling task is due for execution. """ now = time.time() - poll_interval = (self.SHORT_POLL_INTERVAL + poll_interval = (Constants.SHORT_POLL_INTERVAL if now - self._user_stream_tracker.last_recv_time > 60.0 - else self.LONG_POLL_INTERVAL) + else Constants.LONG_POLL_INTERVAL) last_tick = int(self._last_timestamp / poll_interval) current_tick = int(timestamp / poll_interval) if current_tick > last_tick: @@ -791,21 +823,20 @@ async def _user_stream_event_listener(self): """ async for event_message in self._iter_user_event_queue(): try: - if "result" not in event_message or "channel" not in event_message["result"]: + event_methods = [ + Constants.WS_METHODS["USER_ORDERS"], + Constants.WS_METHODS["USER_TRADES"], + ] + method: str = event_message.get("method", None) + params: str = event_message.get("params", None) + + if params is None or method not in event_methods: continue - channel = event_message["result"]["channel"] - if "user.trade" in channel: - for trade_msg in event_message["result"]["data"]: - await self._process_trade_message(trade_msg) - elif "user.order" in channel: - for order_msg in event_message["result"]["data"]: + if method == Constants.WS_METHODS["USER_TRADES"]: + await self._process_trade_message(params) + elif method == Constants.WS_METHODS["USER_ORDERS"]: + for order_msg in params: self._process_order_message(order_msg) - elif channel == "user.balance": - balances = event_message["result"]["data"] - for balance_entry in balances: - asset_name = balance_entry["currency"] - self._account_balances[asset_name] = Decimal(str(balance_entry["balance"])) - self._account_available_balances[asset_name] = Decimal(str(balance_entry["available"])) except asyncio.CancelledError: raise except Exception: diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py b/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py index d51c40ebce..22b9f268d5 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py @@ -21,7 +21,7 @@ def __init__(self, trade_type: TradeType, price: Decimal, amount: Decimal, - initial_state: str = "OPEN"): + initial_state: str = "new"): super().__init__( client_order_id, exchange_order_id, @@ -37,15 +37,15 @@ def __init__(self, @property def is_done(self) -> bool: - return self.last_state in {"FILLED", "CANCELED", "REJECTED", "EXPIRED"} + return self.last_state in {"filled", "canceled", "expired"} @property def is_failure(self) -> bool: - return self.last_state in {"REJECTED"} + return self.last_state in {"suspended"} @property def is_cancelled(self) -> bool: - return self.last_state in {"CANCELED", "EXPIRED"} + return self.last_state in {"canceled", "expired"} # @property # def order_type_description(self) -> str: @@ -83,17 +83,38 @@ def update_with_trade_update(self, trade_update: Dict[str, Any]) -> bool: """ Updates the in flight order with trade update (from private/get-order-detail end point) return: True if the order gets updated otherwise False + Example Trade: + { + "id": "4345697765", + "clientOrderId": "53b7cf917963464a811a4af426102c19", + "symbol": "ETHBTC", + "side": "sell", + "status": "filled", + "type": "limit", + "timeInForce": "GTC", + "quantity": "0.001", + "price": "0.053868", + "cumQuantity": "0.001", + "postOnly": false, + "createdAt": "2017-10-20T12:20:05.952Z", + "updatedAt": "2017-10-20T12:20:38.708Z", + "reportType": "trade", + "tradeQuantity": "0.001", + "tradePrice": "0.053868", + "tradeId": 55051694, + "tradeFee": "-0.000000005" + } """ - trade_id = trade_update["trade_id"] + trade_id = trade_update["clientOrderId"] # trade_update["orderId"] is type int - if str(trade_update["order_id"]) != self.exchange_order_id or trade_id in self.trade_id_set: + if str(trade_update["id"]) != self.exchange_order_id or trade_id in self.trade_id_set: # trade already recorded return False self.trade_id_set.add(trade_id) - self.executed_amount_base += Decimal(str(trade_update["traded_quantity"])) + self.executed_amount_base = Decimal(str(trade_update["cumQuantity"])) self.fee_paid += Decimal(str(trade_update["fee"])) - self.executed_amount_quote += (Decimal(str(trade_update["traded_price"])) * - Decimal(str(trade_update["traded_quantity"]))) + self.executed_amount_quote += (Decimal(str(trade_update["tradePrice"])) * + Decimal(str(trade_update["tradeQuantity"]))) if not self.fee_asset: - self.fee_asset = trade_update["fee_currency"] + self.fee_asset = self.quote_asset return True diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py index d71fbca20d..5f0c67a8a7 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py @@ -111,10 +111,10 @@ def trade_message_from_exchange(cls, msg.update(metadata) msg.update({ - "exchange_order_id": msg.get("d"), - "trade_type": msg.get("s"), - "price": msg.get("p"), - "amount": msg.get("q"), + "exchange_order_id": msg.get("id"), + "trade_type": msg.get("side"), + "price": msg.get("price"), + "amount": msg.get("quantity"), }) return HitBTCOrderBookMessage( diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py index a54c9f1c5d..60db9949de 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py @@ -46,10 +46,10 @@ def trade_id(self) -> int: @property def trading_pair(self) -> str: - if "trading_pair" in self.content: + if "symbol" in self.content: + return self.content["symbol"] + elif "trading_pair" in self.content: return self.content["trading_pair"] - elif "instrument_name" in self.content: - return self.content["instrument_name"] @property def asks(self) -> List[OrderBookRow]: diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py index 5aca798ac3..b1599d1892 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py @@ -4,7 +4,7 @@ import logging import websockets import ujson -import hummingbot.connector.exchange.hitbtc.hitbtc_constants as constants +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants from typing import ( @@ -22,8 +22,6 @@ class HitBTCWebsocket(RequestId): - MESSAGE_TIMEOUT = 30.0 - PING_TIMEOUT = 10.0 _logger: Optional[HummingbotLogger] = None @classmethod @@ -35,7 +33,7 @@ def logger(cls) -> HummingbotLogger: def __init__(self, auth: Optional[HitBTCAuth] = None): self._auth: Optional[HitBTCAuth] = auth self._isPrivate = True if self._auth is not None else False - self._WS_URL = constants.WSS_PRIVATE_URL if self._isPrivate else constants.WSS_PUBLIC_URL + self._WS_URL = Constants.WS_PRIVATE_URL if self._isPrivate else Constants.WS_PUBLIC_URL self._client: Optional[websockets.WebSocketClientProtocol] = None # connect to exchange @@ -67,13 +65,16 @@ async def _messages(self) -> AsyncIterable[Any]: try: while True: try: - raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=self.MESSAGE_TIMEOUT) - raw_msg = ujson.loads(raw_msg_str) - # HitBTC doesn't support ping or heartbeat messages. - # Can handle them here if that changes - use `safe_ensure_future`. - yield raw_msg + raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=Constants.MESSAGE_TIMEOUT) + try: + raw_msg = ujson.loads(raw_msg_str) + # HitBTC doesn't support ping or heartbeat messages. + # Can handle them here if that changes - use `safe_ensure_future`. + yield raw_msg + except ValueError: + continue except asyncio.TimeoutError: - await asyncio.wait_for(self._client.ping(), timeout=self.PING_TIMEOUT) + await asyncio.wait_for(self._client.ping(), timeout=Constants.PING_TIMEOUT) except asyncio.TimeoutError: self.logger().warning("WebSocket ping timed out. Going to reconnect...") return From 256d5c5c66da371a4a8814b4e1d799a1e56bb201 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Fri, 5 Mar 2021 04:37:54 +0000 Subject: [PATCH 08/33] HitBTC: Ready for testing HitBTC: Remove Debugging --- .../hitbtc_api_order_book_data_source.py | 13 +- .../hitbtc_api_user_stream_data_source.py | 3 +- .../exchange/hitbtc/hitbtc_constants.py | 12 ++ .../exchange/hitbtc/hitbtc_exchange.py | 147 ++++++++---------- 4 files changed, 82 insertions(+), 93 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py index c0af9a51eb..17286928a5 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py @@ -107,10 +107,9 @@ async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asynci await ws.connect() for pair in self._trading_pairs: - await ws.subscribe("Trades", convert_to_exchange_trading_pair(pair)) + await ws.subscribe(Constants.WS_SUB["TRADES"], convert_to_exchange_trading_pair(pair)) async for response in ws.on_message(): - print(f"WS1: {response}") method: str = response.get("method", None) trades_data: str = response.get("params", None) @@ -147,16 +146,14 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp await ws.connect() order_book_methods = [ - Constants.WS_METHODS['ORDER_SNAPSHOT'], - Constants.WS_METHODS['ORDER_UPDATE'], + Constants.WS_METHODS['ORDERS_SNAPSHOT'], + Constants.WS_METHODS['ORDERS_UPDATE'], ] for pair in self._trading_pairs: - await ws.subscribe("Orderbook", convert_to_exchange_trading_pair(pair)) + await ws.subscribe(Constants.WS_SUB["ORDERS"], convert_to_exchange_trading_pair(pair)) async for response in ws.on_message(): - print(f"WS2: {response}") - method: str = response.get("method", None) order_book_data: str = response.get("params", None) @@ -167,7 +164,7 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp pair: str = convert_from_exchange_trading_pair(order_book_data["symbol"]) order_book_msg_cls = (HitBTCOrderBook.diff_message_from_exchange - if method == Constants.WS_METHODS['ORDER_UPDATE'] else + if method == Constants.WS_METHODS['ORDERS_UPDATE'] else HitBTCOrderBook.snapshot_message_from_exchange) orderbook_msg: OrderBookMessage = order_book_msg_cls( diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py index 42e33359d8..68fca77a0e 100755 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py @@ -6,6 +6,7 @@ from typing import Optional, List, AsyncIterable, Any from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource from hummingbot.logger import HummingbotLogger +from .hitbtc_constants import Constants from .hitbtc_auth import HitBTCAuth from .hitbtc_websocket import HitBTCWebsocket @@ -40,7 +41,7 @@ async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: try: ws = HitBTCWebsocket(self._hitbtc_auth) await ws.connect() - await ws.subscribe("Reports") + await ws.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"]) async for msg in ws.on_message(): # print(f"WS_SOCKET: {msg}") yield msg diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py index f6cdb5ad5f..1dd8fdcc20 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py @@ -15,6 +15,18 @@ class Constants: "TICKER_SINGLE": "public/ticker/{trading_pair}", "SYMBOL": "public/symbol", "ORDER_BOOK": "public/orderbook", + "ORDER_CREATE": "order", + "ORDER_DELETE": "order/{id}", + "ORDER_STATUS": "order/{id}", + "USER_ORDERS": "order", + "USER_BALANCES": "account/balance", + } + + WS_SUB = { + "TRADES": "Trades", + "ORDERS": "Orderbook", + "USER_ORDERS_TRADES": "Reports", + } WS_METHODS = { diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index 4a5dbcbb2d..400b40451c 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -11,6 +11,7 @@ import aiohttp import math import time +from async_timeout import timeout from hummingbot.core.network_iterator import NetworkStatus from hummingbot.logger import HummingbotLogger @@ -43,6 +44,7 @@ convert_to_exchange_trading_pair, get_new_client_order_id, retry_sleep_time, + str_date_to_ts, HitBTCAPIError, ) from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants @@ -165,7 +167,7 @@ def supported_order_types(self) -> List[OrderType]: :return a list of OrderType supported by this connector. Note that Market order type is no longer required and will not be used. """ - return [OrderType.LIMIT] + return [OrderType.LIMIT, OrderType.LIMIT_MAKER] def start(self, clock: Clock, timestamp: float): """ @@ -349,6 +351,8 @@ async def _api_request(self, self.logger().network(f"Error fetching data from {url}. HTTP status is {http_status}. " f"Final msg: {parsed_response}.") raise HitBTCAPIError({"error": parsed_response, "status": http_status}) + if "error" in parsed_response: + raise HitBTCAPIError(parsed_response) return parsed_response def get_order_price_quantum(self, trading_pair: str, price: Decimal): @@ -435,53 +439,45 @@ async def _create_order(self, if amount < trading_rule.min_order_size: raise ValueError(f"Buy order amount {amount} is lower than the minimum order size " f"{trading_rule.min_order_size}.") - api_params = {"instrument_name": convert_to_exchange_trading_pair(trading_pair), - "side": trade_type.name, - "type": "LIMIT", + order_type_str = order_type.name.lower().split("_")[0] + api_params = {"symbol": convert_to_exchange_trading_pair(trading_pair), + "side": trade_type.name.lower(), + "type": order_type_str, "price": f"{price:f}", "quantity": f"{amount:f}", - "client_oid": order_id + "clientOrderId": order_id, + # Without strict validate, HitBTC might adjust order prices/sizes. + "strictValidate": "true", } if order_type is OrderType.LIMIT_MAKER: - api_params["exec_inst"] = "POST_ONLY" - self.start_tracking_order(order_id, - None, - trading_pair, - trade_type, - price, - amount, - order_type - ) + api_params["postOnly"] = "true" + self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, order_type) try: - order_result = await self._api_request("post", "private/create-order", api_params, True) - exchange_order_id = str(order_result["result"]["order_id"]) + order_result = await self._api_request("post", Constants.ENDPOINT["ORDER_CREATE"], api_params, True) + exchange_order_id = str(order_result["id"]) tracked_order = self._in_flight_orders.get(order_id) if tracked_order is not None: self.logger().info(f"Created {order_type.name} {trade_type.name} order {order_id} for " f"{amount} {trading_pair}.") tracked_order.update_exchange_order_id(exchange_order_id) - - event_tag = MarketEvent.BuyOrderCreated if trade_type is TradeType.BUY else MarketEvent.SellOrderCreated - event_class = BuyOrderCreatedEvent if trade_type is TradeType.BUY else SellOrderCreatedEvent + if trade_type is TradeType.BUY: + event_tag = MarketEvent.BuyOrderCreated + event_cls = BuyOrderCreatedEvent + else: + event_tag = MarketEvent.SellOrderCreated + event_cls = SellOrderCreatedEvent self.trigger_event(event_tag, - event_class( - self.current_timestamp, - order_type, - trading_pair, - amount, - price, - order_id - )) + event_cls(self.current_timestamp, order_type, trading_pair, amount, price, order_id)) except asyncio.CancelledError: raise - except Exception as e: + except HitBTCAPIError as e: + error_reason = str(e.error_payload['error']) self.stop_tracking_order(order_id) self.logger().network( f"Error submitting {trade_type.name} {order_type.name} order to HitBTC for " - f"{amount} {trading_pair} " - f"{price}.", + f"{amount} {trading_pair} {price} - {error_reason}.", exc_info=True, - app_warning_msg=str(e) + app_warning_msg=(f"Error submitting order to HitBTC - {error_reason}.") ) self.trigger_event(MarketEvent.OrderFailure, MarketOrderFailureEvent(self.current_timestamp, order_id, order_type)) @@ -528,24 +524,20 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: raise ValueError(f"Failed to cancel order - {order_id}. Order not found.") if tracked_order.exchange_order_id is None: await tracked_order.get_exchange_order_id() - ex_order_id = tracked_order.exchange_order_id - await self._api_request( - "post", - "private/cancel-order", - {"instrument_name": convert_to_exchange_trading_pair(trading_pair), - "order_id": ex_order_id}, - True - ) - return order_id + # ex_order_id = tracked_order.exchange_order_id + await self._api_request("delete", Constants.ENDPOINT["ORDER_DELETE"].format(id=order_id), True) + return CancellationResult(order_id, True) except asyncio.CancelledError: raise - except Exception as e: + except HitBTCAPIError as e: + error_reason = str(e.error_payload['error']) self.logger().network( - f"Failed to cancel order {order_id}: {str(e)}", + f"Failed to cancel order {order_id}: {error_reason}", exc_info=True, app_warning_msg=f"Failed to cancel the order {order_id} on HitBTC. " f"Check API key and network connection." ) + return CancellationResult(order_id, False) async def _status_polling_loop(self): """ @@ -577,11 +569,11 @@ async def _update_balances(self): """ local_asset_names = set(self._account_balances.keys()) remote_asset_names = set() - account_info = await self._api_request("post", "private/get-account-summary", {}, True) - for account in account_info["result"]["accounts"]: + account_info = await self._api_request("get", Constants.ENDPOINT["USER_BALANCES"], is_auth_required=True) + for account in account_info: asset_name = account["currency"] self._account_available_balances[asset_name] = Decimal(str(account["available"])) - self._account_balances[asset_name] = Decimal(str(account["balance"])) + self._account_balances[asset_name] = Decimal(str(account["reserved"])) + Decimal(str(account["available"])) remote_asset_names.add(asset_name) asset_names_to_remove = local_asset_names.difference(remote_asset_names) @@ -600,11 +592,11 @@ async def _update_order_status(self): tracked_orders = list(self._in_flight_orders.values()) tasks = [] for tracked_order in tracked_orders: - order_id = await tracked_order.get_exchange_order_id() - tasks.append(self._api_request("post", - "private/get-order-detail", - {"order_id": order_id}, - True)) + # exchange_order_id = await tracked_order.get_exchange_order_id() + order_id = tracked_order.client_order_id + tasks.append(self._api_request("get", + Constants.ENDPOINT["ORDER_STATUS"].format(id=order_id), + is_auth_required=True)) self.logger().debug(f"Polling for order status updates of {len(tasks)} orders.") responses = await safe_gather(*tasks, return_exceptions=True) for response in responses: @@ -736,7 +728,7 @@ async def _process_trade_message(self, trade_msg: Dict[str, Any]): tracked_order.order_type)) self.stop_tracking_order(tracked_order.client_order_id) - async def cancel_all(self, timeout_seconds: float): + async def cancel_all(self, timeout_seconds: float) -> List[CancellationResult]: """ Cancels all in-flight orders and waits for cancellation results. Used by bot's top level stop and exit commands (cancelling outstanding orders on exit) @@ -745,27 +737,17 @@ async def cancel_all(self, timeout_seconds: float): """ if self._trading_pairs is None: raise Exception("cancel_all can only be used when trading_pairs are specified.") + open_orders = [o for o in self._in_flight_orders.values() if not o.is_done] + if len(open_orders) == 0: + return [] + tasks = [self._execute_cancel(o.client_order_id) for o in open_orders] cancellation_results = [] try: - for trading_pair in self._trading_pairs: - await self._api_request( - "post", - "private/cancel-all-orders", - {"instrument_name": convert_to_exchange_trading_pair(trading_pair)}, - True - ) - open_orders = await self.get_open_orders() - for cl_order_id, tracked_order in self._in_flight_orders.items(): - open_order = [o for o in open_orders if o.client_order_id == cl_order_id] - if not open_order: - cancellation_results.append(CancellationResult(cl_order_id, True)) - self.trigger_event(MarketEvent.OrderCancelled, - OrderCancelledEvent(self.current_timestamp, cl_order_id)) - else: - cancellation_results.append(CancellationResult(cl_order_id, False)) + async with timeout(timeout_seconds): + cancellation_results = await safe_gather(*tasks, return_exceptions=True) except Exception: self.logger().network( - "Failed to cancel all orders.", + "Unexpected error cancelling orders.", exc_info=True, app_warning_msg="Failed to cancel all orders on HitBTC. Check API key and network connection." ) @@ -843,31 +825,28 @@ async def _user_stream_event_listener(self): self.logger().error("Unexpected error in user stream listener loop.", exc_info=True) await asyncio.sleep(5.0) + # This is currently unused, but looks like a future addition. async def get_open_orders(self) -> List[OpenOrder]: - result = await self._api_request( - "post", - "private/get-open-orders", - {}, - True - ) + result = await self._api_request("get", Constants.ENDPOINT["USER_ORDERS"], is_auth_required=True) ret_val = [] - for order in result["result"]["order_list"]: - if Constants.HBOT_BROKER_ID not in order["client_oid"]: + for order in result: + if Constants.HBOT_BROKER_ID not in order["clientOrderId"]: + continue + if order["type"] != OrderType.LIMIT.name.lower(): + self.logger().info(f"Unsupported order type found: {order['type']}") continue - if order["type"] != "LIMIT": - raise Exception(f"Unsupported order type {order['type']}") ret_val.append( OpenOrder( - client_order_id=order["client_oid"], - trading_pair=convert_from_exchange_trading_pair(order["instrument_name"]), + client_order_id=order["clientOrderId"], + trading_pair=convert_from_exchange_trading_pair(order["symbol"]), price=Decimal(str(order["price"])), amount=Decimal(str(order["quantity"])), - executed_amount=Decimal(str(order["cumulative_quantity"])), + executed_amount=Decimal(str(order["cumQuantity"])), status=order["status"], order_type=OrderType.LIMIT, - is_buy=True if order["side"].lower() == "buy" else False, - time=int(order["create_time"]), - exchange_order_id=order["order_id"] + is_buy=True if order["side"].lower() == TradeType.BUY.name.lower() else False, + time=str_date_to_ts(order["createdAt"]), + exchange_order_id=order["id"] ) ) return ret_val From b7d6c1e4a40f1ef3b4bd3fe9705a3d4cf7366f24 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Sun, 7 Mar 2021 11:51:28 +0000 Subject: [PATCH 09/33] HitBTC: Fix class/method cases --- .../hitbtc/hitbtc_active_order_tracker.pxd | 2 +- .../hitbtc/hitbtc_active_order_tracker.pyx | 12 +++--- .../hitbtc_api_order_book_data_source.py | 28 +++++++------- .../hitbtc_api_user_stream_data_source.py | 12 +++--- .../connector/exchange/hitbtc/hitbtc_auth.py | 2 +- .../exchange/hitbtc/hitbtc_exchange.py | 38 +++++++++---------- .../exchange/hitbtc/hitbtc_in_flight_order.py | 4 +- .../exchange/hitbtc/hitbtc_order_book.py | 28 +++++++------- .../hitbtc/hitbtc_order_book_message.py | 4 +- .../hitbtc/hitbtc_order_book_tracker.py | 30 +++++++-------- .../hitbtc/hitbtc_order_book_tracker_entry.py | 12 +++--- .../hitbtc/hitbtc_user_stream_tracker.py | 12 +++--- .../connector/exchange/hitbtc/hitbtc_utils.py | 4 +- .../exchange/hitbtc/hitbtc_websocket.py | 8 ++-- 14 files changed, 98 insertions(+), 98 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pxd b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pxd index 68d8102dd0..97c2af5b02 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pxd +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pxd @@ -1,7 +1,7 @@ # distutils: language=c++ cimport numpy as np -cdef class HitBTCActiveOrderTracker: +cdef class HitbtcActiveOrderTracker: cdef dict _active_bids cdef dict _active_asks diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx index 6ba7d6a3be..248206f060 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx @@ -11,12 +11,12 @@ from hummingbot.core.data_type.order_book_row import OrderBookRow _logger = None s_empty_diff = np.ndarray(shape=(0, 4), dtype="float64") -HitBTCOrderBookTrackingDictionary = Dict[Decimal, Dict[str, Dict[str, any]]] +HitbtcOrderBookTrackingDictionary = Dict[Decimal, Dict[str, Dict[str, any]]] -cdef class HitBTCActiveOrderTracker: +cdef class HitbtcActiveOrderTracker: def __init__(self, - active_asks: HitBTCOrderBookTrackingDictionary = None, - active_bids: HitBTCOrderBookTrackingDictionary = None): + active_asks: HitbtcOrderBookTrackingDictionary = None, + active_bids: HitbtcOrderBookTrackingDictionary = None): super().__init__() self._active_asks = active_asks or {} self._active_bids = active_bids or {} @@ -29,11 +29,11 @@ cdef class HitBTCActiveOrderTracker: return _logger @property - def active_asks(self) -> HitBTCOrderBookTrackingDictionary: + def active_asks(self) -> HitbtcOrderBookTrackingDictionary: return self._active_asks @property - def active_bids(self) -> HitBTCOrderBookTrackingDictionary: + def active_bids(self) -> HitbtcOrderBookTrackingDictionary: return self._active_bids # TODO: research this more diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py index 17286928a5..3f2369313a 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py @@ -11,19 +11,19 @@ # from hummingbot.core.utils.async_utils import safe_gather from hummingbot.logger import HummingbotLogger from .hitbtc_constants import Constants -from .hitbtc_active_order_tracker import HitBTCActiveOrderTracker -from .hitbtc_order_book import HitBTCOrderBook -from .hitbtc_websocket import HitBTCWebsocket +from .hitbtc_active_order_tracker import HitbtcActiveOrderTracker +from .hitbtc_order_book import HitbtcOrderBook +from .hitbtc_websocket import HitbtcWebsocket from .hitbtc_utils import ( str_date_to_ts, convert_to_exchange_trading_pair, convert_from_exchange_trading_pair, generic_api_request, - HitBTCAPIError, + HitbtcAPIError, ) -class HitBTCAPIOrderBookDataSource(OrderBookTrackerDataSource): +class HitbtcAPIOrderBookDataSource(OrderBookTrackerDataSource): _logger: Optional[HummingbotLogger] = None @classmethod @@ -77,7 +77,7 @@ async def get_order_book_data(trading_pair: str) -> Dict[str, any]: }) orderbook_data = orderbook_response[ex_pair] return orderbook_data - except HitBTCAPIError as e: + except HitbtcAPIError as e: raise IOError( f"Error fetching OrderBook for {trading_pair} at {Constants.EXCHANGE_NAME}. " f"HTTP status is {e.error_payload['status']}. Error is {e.error_payload['error']}." @@ -86,13 +86,13 @@ async def get_order_book_data(trading_pair: str) -> Dict[str, any]: async def get_new_order_book(self, trading_pair: str) -> OrderBook: snapshot: Dict[str, Any] = await self.get_order_book_data(trading_pair) snapshot_timestamp: float = time.time() - snapshot_msg: OrderBookMessage = HitBTCOrderBook.snapshot_message_from_exchange( + snapshot_msg: OrderBookMessage = HitbtcOrderBook.snapshot_message_from_exchange( snapshot, snapshot_timestamp, metadata={"trading_pair": trading_pair} ) order_book = self.order_book_create_function() - active_order_tracker: HitBTCActiveOrderTracker = HitBTCActiveOrderTracker() + active_order_tracker: HitbtcActiveOrderTracker = HitbtcActiveOrderTracker() bids, asks = active_order_tracker.convert_snapshot_message_to_order_book_row(snapshot_msg) order_book.apply_snapshot(bids, asks, snapshot_msg.update_id) return order_book @@ -103,7 +103,7 @@ async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asynci """ while True: try: - ws = HitBTCWebsocket() + ws = HitbtcWebsocket() await ws.connect() for pair in self._trading_pairs: @@ -121,7 +121,7 @@ async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asynci for trade in trades_data["data"]: trade: Dict[Any] = trade trade_timestamp: int = str_date_to_ts(trade["timestamp"]) - trade_msg: OrderBookMessage = HitBTCOrderBook.trade_message_from_exchange( + trade_msg: OrderBookMessage = HitbtcOrderBook.trade_message_from_exchange( trade, trade_timestamp, metadata={"trading_pair": pair} @@ -142,7 +142,7 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp """ while True: try: - ws = HitBTCWebsocket() + ws = HitbtcWebsocket() await ws.connect() order_book_methods = [ @@ -163,9 +163,9 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp timestamp: int = str_date_to_ts(order_book_data["timestamp"]) pair: str = convert_from_exchange_trading_pair(order_book_data["symbol"]) - order_book_msg_cls = (HitBTCOrderBook.diff_message_from_exchange + order_book_msg_cls = (HitbtcOrderBook.diff_message_from_exchange if method == Constants.WS_METHODS['ORDERS_UPDATE'] else - HitBTCOrderBook.snapshot_message_from_exchange) + HitbtcOrderBook.snapshot_message_from_exchange) orderbook_msg: OrderBookMessage = order_book_msg_cls( order_book_data, @@ -197,7 +197,7 @@ async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, try: snapshot: Dict[str, any] = await self.get_order_book_data(trading_pair) snapshot_timestamp: int = str_date_to_ts(snapshot["timestamp"]) - snapshot_msg: OrderBookMessage = HitBTCOrderBook.snapshot_message_from_exchange( + snapshot_msg: OrderBookMessage = HitbtcOrderBook.snapshot_message_from_exchange( snapshot, snapshot_timestamp, metadata={"trading_pair": trading_pair} diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py index 68fca77a0e..829ee25940 100755 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py @@ -7,11 +7,11 @@ from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource from hummingbot.logger import HummingbotLogger from .hitbtc_constants import Constants -from .hitbtc_auth import HitBTCAuth -from .hitbtc_websocket import HitBTCWebsocket +from .hitbtc_auth import HitbtcAuth +from .hitbtc_websocket import HitbtcWebsocket -class HitBTCAPIUserStreamDataSource(UserStreamTrackerDataSource): +class HitbtcAPIUserStreamDataSource(UserStreamTrackerDataSource): _logger: Optional[HummingbotLogger] = None @@ -21,8 +21,8 @@ def logger(cls) -> HummingbotLogger: cls._logger = logging.getLogger(__name__) return cls._logger - def __init__(self, hitbtc_auth: HitBTCAuth, trading_pairs: Optional[List[str]] = []): - self._hitbtc_auth: HitBTCAuth = hitbtc_auth + def __init__(self, hitbtc_auth: HitbtcAuth, trading_pairs: Optional[List[str]] = []): + self._hitbtc_auth: HitbtcAuth = hitbtc_auth self._trading_pairs = trading_pairs self._current_listen_key = None self._listen_for_user_stream_task = None @@ -39,7 +39,7 @@ async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: """ try: - ws = HitBTCWebsocket(self._hitbtc_auth) + ws = HitbtcWebsocket(self._hitbtc_auth) await ws.connect() await ws.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"]) async for msg in ws.on_message(): diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py b/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py index 8f0159aa6f..5b54054478 100755 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py @@ -6,7 +6,7 @@ from typing import Dict, Any -class HitBTCAuth(): +class HitbtcAuth(): """ Auth class required by HitBTC API Learn more at https://exchange-docs.crypto.com/#digital-signature diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index 400b40451c..85cc08c30e 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -35,17 +35,17 @@ TradeFee ) from hummingbot.connector.exchange_base import ExchangeBase -from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_tracker import HitBTCOrderBookTracker -from hummingbot.connector.exchange.hitbtc.hitbtc_user_stream_tracker import HitBTCUserStreamTracker -from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitBTCAuth -from hummingbot.connector.exchange.hitbtc.hitbtc_in_flight_order import HitBTCInFlightOrder +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_tracker import HitbtcOrderBookTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_user_stream_tracker import HitbtcUserStreamTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth +from hummingbot.connector.exchange.hitbtc.hitbtc_in_flight_order import HitbtcInFlightOrder from hummingbot.connector.exchange.hitbtc.hitbtc_utils import ( convert_from_exchange_trading_pair, convert_to_exchange_trading_pair, get_new_client_order_id, retry_sleep_time, str_date_to_ts, - HitBTCAPIError, + HitbtcAPIError, ) from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants from hummingbot.core.data_type.common import OpenOrder @@ -53,9 +53,9 @@ s_decimal_NaN = Decimal("nan") -class HitBTCExchange(ExchangeBase): +class HitbtcExchange(ExchangeBase): """ - HitBTCExchange connects with HitBTC exchange and provides order book pricing, user account tracking and + HitbtcExchange connects with HitBTC exchange and provides order book pricing, user account tracking and trading functionality. """ @@ -81,14 +81,14 @@ def __init__(self, super().__init__() self._trading_required = trading_required self._trading_pairs = trading_pairs - self._hitbtc_auth = HitBTCAuth(hitbtc_api_key, hitbtc_secret_key) - self._order_book_tracker = HitBTCOrderBookTracker(trading_pairs=trading_pairs) - self._user_stream_tracker = HitBTCUserStreamTracker(self._hitbtc_auth, trading_pairs) + self._hitbtc_auth = HitbtcAuth(hitbtc_api_key, hitbtc_secret_key) + self._order_book_tracker = HitbtcOrderBookTracker(trading_pairs=trading_pairs) + self._user_stream_tracker = HitbtcUserStreamTracker(self._hitbtc_auth, trading_pairs) self._ev_loop = asyncio.get_event_loop() self._shared_client = None self._poll_notifier = asyncio.Event() self._last_timestamp = 0 - self._in_flight_orders = {} # Dict[client_order_id:str, HitBTCInFlightOrder] + self._in_flight_orders = {} # Dict[client_order_id:str, HitbtcInFlightOrder] self._order_not_found_records = {} # Dict[client_order_id:str, count:int] self._trading_rules = {} # Dict[trading_pair:str, TradingRule] self._status_polling_task = None @@ -109,7 +109,7 @@ def trading_rules(self) -> Dict[str, TradingRule]: return self._trading_rules @property - def in_flight_orders(self) -> Dict[str, HitBTCInFlightOrder]: + def in_flight_orders(self) -> Dict[str, HitbtcInFlightOrder]: return self._in_flight_orders @property @@ -158,7 +158,7 @@ def restore_tracking_states(self, saved_states: Dict[str, any]): :param saved_states: The saved tracking_states. """ self._in_flight_orders.update({ - key: HitBTCInFlightOrder.from_json(value) + key: HitbtcInFlightOrder.from_json(value) for key, value in saved_states.items() }) @@ -350,9 +350,9 @@ async def _api_request(self, else: self.logger().network(f"Error fetching data from {url}. HTTP status is {http_status}. " f"Final msg: {parsed_response}.") - raise HitBTCAPIError({"error": parsed_response, "status": http_status}) + raise HitbtcAPIError({"error": parsed_response, "status": http_status}) if "error" in parsed_response: - raise HitBTCAPIError(parsed_response) + raise HitbtcAPIError(parsed_response) return parsed_response def get_order_price_quantum(self, trading_pair: str, price: Decimal): @@ -470,7 +470,7 @@ async def _create_order(self, event_cls(self.current_timestamp, order_type, trading_pair, amount, price, order_id)) except asyncio.CancelledError: raise - except HitBTCAPIError as e: + except HitbtcAPIError as e: error_reason = str(e.error_payload['error']) self.stop_tracking_order(order_id) self.logger().network( @@ -493,7 +493,7 @@ def start_tracking_order(self, """ Starts tracking an order by simply adding it into _in_flight_orders dictionary. """ - self._in_flight_orders[order_id] = HitBTCInFlightOrder( + self._in_flight_orders[order_id] = HitbtcInFlightOrder( client_order_id=order_id, exchange_order_id=exchange_order_id, trading_pair=trading_pair, @@ -529,7 +529,7 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: return CancellationResult(order_id, True) except asyncio.CancelledError: raise - except HitBTCAPIError as e: + except HitbtcAPIError as e: error_reason = str(e.error_payload['error']) self.logger().network( f"Failed to cancel order {order_id}: {error_reason}", @@ -801,7 +801,7 @@ async def _iter_user_event_queue(self) -> AsyncIterable[Dict[str, any]]: async def _user_stream_event_listener(self): """ Listens to message in _user_stream_tracker.user_stream queue. The messages are put in by - HitBTCAPIUserStreamDataSource. + HitbtcAPIUserStreamDataSource. """ async for event_message in self._iter_user_event_queue(): try: diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py b/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py index 22b9f268d5..8654dddfaa 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py @@ -12,7 +12,7 @@ from hummingbot.connector.in_flight_order_base import InFlightOrderBase -class HitBTCInFlightOrder(InFlightOrderBase): +class HitbtcInFlightOrder(InFlightOrderBase): def __init__(self, client_order_id: str, exchange_order_id: Optional[str], @@ -62,7 +62,7 @@ def from_json(cls, data: Dict[str, Any]) -> InFlightOrderBase: :param data: json data from API :return: formatted InFlightOrder """ - retval = HitBTCInFlightOrder( + retval = HitbtcInFlightOrder( data["client_order_id"], data["exchange_order_id"], data["trading_pair"], diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py index 5f0c67a8a7..711beaa961 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py @@ -13,12 +13,12 @@ from hummingbot.core.data_type.order_book_message import ( OrderBookMessage, OrderBookMessageType ) -from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_message import HitBTCOrderBookMessage +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_message import HitbtcOrderBookMessage _logger = None -class HitBTCOrderBook(OrderBook): +class HitbtcOrderBook(OrderBook): @classmethod def logger(cls) -> HummingbotLogger: global _logger @@ -35,13 +35,13 @@ def snapshot_message_from_exchange(cls, Convert json snapshot data into standard OrderBookMessage format :param msg: json snapshot data from live web socket stream :param timestamp: timestamp attached to incoming data - :return: HitBTCOrderBookMessage + :return: HitbtcOrderBookMessage """ if metadata: msg.update(metadata) - return HitBTCOrderBookMessage( + return HitbtcOrderBookMessage( message_type=OrderBookMessageType.SNAPSHOT, content=msg, timestamp=timestamp @@ -53,9 +53,9 @@ def snapshot_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = N *used for backtesting Convert a row of snapshot data into standard OrderBookMessage format :param record: a row of snapshot data from the database - :return: HitBTCOrderBookMessage + :return: HitbtcOrderBookMessage """ - return HitBTCOrderBookMessage( + return HitbtcOrderBookMessage( message_type=OrderBookMessageType.SNAPSHOT, content=record.json, timestamp=record.timestamp @@ -70,13 +70,13 @@ def diff_message_from_exchange(cls, Convert json diff data into standard OrderBookMessage format :param msg: json diff data from live web socket stream :param timestamp: timestamp attached to incoming data - :return: HitBTCOrderBookMessage + :return: HitbtcOrderBookMessage """ if metadata: msg.update(metadata) - return HitBTCOrderBookMessage( + return HitbtcOrderBookMessage( message_type=OrderBookMessageType.DIFF, content=msg, timestamp=timestamp @@ -88,9 +88,9 @@ def diff_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None) *used for backtesting Convert a row of diff data into standard OrderBookMessage format :param record: a row of diff data from the database - :return: HitBTCOrderBookMessage + :return: HitbtcOrderBookMessage """ - return HitBTCOrderBookMessage( + return HitbtcOrderBookMessage( message_type=OrderBookMessageType.DIFF, content=record.json, timestamp=record.timestamp @@ -104,7 +104,7 @@ def trade_message_from_exchange(cls, """ Convert a trade data into standard OrderBookMessage format :param record: a trade data from the database - :return: HitBTCOrderBookMessage + :return: HitbtcOrderBookMessage """ if metadata: @@ -117,7 +117,7 @@ def trade_message_from_exchange(cls, "amount": msg.get("quantity"), }) - return HitBTCOrderBookMessage( + return HitbtcOrderBookMessage( message_type=OrderBookMessageType.TRADE, content=msg, timestamp=timestamp @@ -129,9 +129,9 @@ def trade_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None *used for backtesting Convert a row of trade data into standard OrderBookMessage format :param record: a row of trade data from the database - :return: HitBTCOrderBookMessage + :return: HitbtcOrderBookMessage """ - return HitBTCOrderBookMessage( + return HitbtcOrderBookMessage( message_type=OrderBookMessageType.TRADE, content=record.json, timestamp=record.timestamp diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py index 60db9949de..8658c56acf 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py @@ -13,7 +13,7 @@ ) -class HitBTCOrderBookMessage(OrderBookMessage): +class HitbtcOrderBookMessage(OrderBookMessage): def __new__( cls, message_type: OrderBookMessageType, @@ -27,7 +27,7 @@ def __new__( raise ValueError("timestamp must not be None when initializing snapshot messages.") timestamp = content["timestamp"] - return super(HitBTCOrderBookMessage, cls).__new__( + return super(HitbtcOrderBookMessage, cls).__new__( cls, message_type, content, timestamp=timestamp, *args, **kwargs ) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py index 39f5850bfa..19707fe7dc 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py @@ -10,13 +10,13 @@ from hummingbot.core.data_type.order_book_message import OrderBookMessageType from hummingbot.logger import HummingbotLogger from hummingbot.core.data_type.order_book_tracker import OrderBookTracker -from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_message import HitBTCOrderBookMessage -from hummingbot.connector.exchange.hitbtc.hitbtc_active_order_tracker import HitBTCActiveOrderTracker -from hummingbot.connector.exchange.hitbtc.hitbtc_api_order_book_data_source import HitBTCAPIOrderBookDataSource -from hummingbot.connector.exchange.hitbtc.hitbtc_order_book import HitBTCOrderBook +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_message import HitbtcOrderBookMessage +from hummingbot.connector.exchange.hitbtc.hitbtc_active_order_tracker import HitbtcActiveOrderTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_api_order_book_data_source import HitbtcAPIOrderBookDataSource +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book import HitbtcOrderBook -class HitBTCOrderBookTracker(OrderBookTracker): +class HitbtcOrderBookTracker(OrderBookTracker): _logger: Optional[HummingbotLogger] = None @classmethod @@ -26,7 +26,7 @@ def logger(cls) -> HummingbotLogger: return cls._logger def __init__(self, trading_pairs: Optional[List[str]] = None,): - super().__init__(HitBTCAPIOrderBookDataSource(trading_pairs), trading_pairs) + super().__init__(HitbtcAPIOrderBookDataSource(trading_pairs), trading_pairs) self._ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() self._order_book_snapshot_stream: asyncio.Queue = asyncio.Queue() @@ -34,10 +34,10 @@ def __init__(self, trading_pairs: Optional[List[str]] = None,): self._order_book_trade_stream: asyncio.Queue = asyncio.Queue() self._process_msg_deque_task: Optional[asyncio.Task] = None self._past_diffs_windows: Dict[str, Deque] = {} - self._order_books: Dict[str, HitBTCOrderBook] = {} - self._saved_message_queues: Dict[str, Deque[HitBTCOrderBookMessage]] = \ + self._order_books: Dict[str, HitbtcOrderBook] = {} + self._saved_message_queues: Dict[str, Deque[HitbtcOrderBookMessage]] = \ defaultdict(lambda: deque(maxlen=1000)) - self._active_order_trackers: Dict[str, HitBTCActiveOrderTracker] = defaultdict(HitBTCActiveOrderTracker) + self._active_order_trackers: Dict[str, HitbtcActiveOrderTracker] = defaultdict(HitbtcActiveOrderTracker) self._order_book_stream_listener_task: Optional[asyncio.Task] = None self._order_book_trade_listener_task: Optional[asyncio.Task] = None @@ -52,20 +52,20 @@ async def _track_single_book(self, trading_pair: str): """ Update an order book with changes from the latest batch of received messages """ - past_diffs_window: Deque[HitBTCOrderBookMessage] = deque() + past_diffs_window: Deque[HitbtcOrderBookMessage] = deque() self._past_diffs_windows[trading_pair] = past_diffs_window message_queue: asyncio.Queue = self._tracking_message_queues[trading_pair] - order_book: HitBTCOrderBook = self._order_books[trading_pair] - active_order_tracker: HitBTCActiveOrderTracker = self._active_order_trackers[trading_pair] + order_book: HitbtcOrderBook = self._order_books[trading_pair] + active_order_tracker: HitbtcActiveOrderTracker = self._active_order_trackers[trading_pair] last_message_timestamp: float = time.time() diff_messages_accepted: int = 0 while True: try: - message: HitBTCOrderBookMessage = None - saved_messages: Deque[HitBTCOrderBookMessage] = self._saved_message_queues[trading_pair] + message: HitbtcOrderBookMessage = None + saved_messages: Deque[HitbtcOrderBookMessage] = self._saved_message_queues[trading_pair] # Process saved messages first if there are any if len(saved_messages) > 0: message = saved_messages.popleft() @@ -88,7 +88,7 @@ async def _track_single_book(self, trading_pair: str): diff_messages_accepted = 0 last_message_timestamp = now elif message.type is OrderBookMessageType.SNAPSHOT: - past_diffs: List[HitBTCOrderBookMessage] = list(past_diffs_window) + past_diffs: List[HitbtcOrderBookMessage] = list(past_diffs_window) # only replay diffs later than snapshot, first update active order with snapshot then replay diffs replay_position = bisect.bisect_right(past_diffs, message) replay_diffs = past_diffs[replay_position:] diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker_entry.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker_entry.py index d463f3549c..5edfbadec0 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker_entry.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker_entry.py @@ -1,21 +1,21 @@ from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.order_book_tracker_entry import OrderBookTrackerEntry -from hummingbot.connector.exchange.hitbtc.hitbtc_active_order_tracker import HitBTCActiveOrderTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_active_order_tracker import HitbtcActiveOrderTracker -class HitBTCOrderBookTrackerEntry(OrderBookTrackerEntry): +class HitbtcOrderBookTrackerEntry(OrderBookTrackerEntry): def __init__( - self, trading_pair: str, timestamp: float, order_book: OrderBook, active_order_tracker: HitBTCActiveOrderTracker + self, trading_pair: str, timestamp: float, order_book: OrderBook, active_order_tracker: HitbtcActiveOrderTracker ): self._active_order_tracker = active_order_tracker - super(HitBTCOrderBookTrackerEntry, self).__init__(trading_pair, timestamp, order_book) + super(HitbtcOrderBookTrackerEntry, self).__init__(trading_pair, timestamp, order_book) def __repr__(self) -> str: return ( - f"HitBTCOrderBookTrackerEntry(trading_pair='{self._trading_pair}', timestamp='{self._timestamp}', " + f"HitbtcOrderBookTrackerEntry(trading_pair='{self._trading_pair}', timestamp='{self._timestamp}', " f"order_book='{self._order_book}')" ) @property - def active_order_tracker(self) -> HitBTCActiveOrderTracker: + def active_order_tracker(self) -> HitbtcActiveOrderTracker: return self._active_order_tracker diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_user_stream_tracker.py b/hummingbot/connector/exchange/hitbtc/hitbtc_user_stream_tracker.py index a3b54eb2fa..6e17029209 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_user_stream_tracker.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_user_stream_tracker.py @@ -16,12 +16,12 @@ safe_gather, ) from hummingbot.connector.exchange.hitbtc.hitbtc_api_user_stream_data_source import \ - HitBTCAPIUserStreamDataSource -from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitBTCAuth + HitbtcAPIUserStreamDataSource +from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth from hummingbot.connector.exchange.hitbtc.hitbtc_constants import EXCHANGE_NAME -class HitBTCUserStreamTracker(UserStreamTracker): +class HitbtcUserStreamTracker(UserStreamTracker): _cbpust_logger: Optional[HummingbotLogger] = None @classmethod @@ -31,10 +31,10 @@ def logger(cls) -> HummingbotLogger: return cls._bust_logger def __init__(self, - hitbtc_auth: Optional[HitBTCAuth] = None, + hitbtc_auth: Optional[HitbtcAuth] = None, trading_pairs: Optional[List[str]] = []): super().__init__() - self._hitbtc_auth: HitBTCAuth = hitbtc_auth + self._hitbtc_auth: HitbtcAuth = hitbtc_auth self._trading_pairs: List[str] = trading_pairs self._ev_loop: asyncio.events.AbstractEventLoop = asyncio.get_event_loop() self._data_source: Optional[UserStreamTrackerDataSource] = None @@ -48,7 +48,7 @@ def data_source(self) -> UserStreamTrackerDataSource: :return: OrderBookTrackerDataSource """ if not self._data_source: - self._data_source = HitBTCAPIUserStreamDataSource( + self._data_source = HitbtcAPIUserStreamDataSource( hitbtc_auth=self._hitbtc_auth, trading_pairs=self._trading_pairs ) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py index a8ef8ade17..1e1b67cd9d 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py @@ -27,7 +27,7 @@ DEFAULT_FEES = [0.1, 0.1] -class HitBTCAPIError(IOError): +class HitbtcAPIError(IOError): def __init__(self, error_payload: Dict[str, Any]): super().__init__(str(error_payload)) self.error_payload = error_payload @@ -153,7 +153,7 @@ async def generic_api_request(method, else: print(f"Error fetching data from {url}. HTTP status is {http_status}. " f"Final msg: {parsed_response}.") - raise HitBTCAPIError({"error": parsed_response, "status": http_status}) + raise HitbtcAPIError({"error": parsed_response, "status": http_status}) return parsed_response diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py index b1599d1892..2f72c260eb 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py @@ -14,14 +14,14 @@ ) from websockets.exceptions import ConnectionClosed from hummingbot.logger import HummingbotLogger -from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitBTCAuth +from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth from hummingbot.connector.exchange.hitbtc.hitbtc_utils import RequestId # reusable websocket class # ToDo: We should eventually remove this class, and instantiate web socket connection normally (see Binance for example) -class HitBTCWebsocket(RequestId): +class HitbtcWebsocket(RequestId): _logger: Optional[HummingbotLogger] = None @classmethod @@ -30,8 +30,8 @@ def logger(cls) -> HummingbotLogger: cls._logger = logging.getLogger(__name__) return cls._logger - def __init__(self, auth: Optional[HitBTCAuth] = None): - self._auth: Optional[HitBTCAuth] = auth + def __init__(self, auth: Optional[HitbtcAuth] = None): + self._auth: Optional[HitbtcAuth] = auth self._isPrivate = True if self._auth is not None else False self._WS_URL = Constants.WS_PRIVATE_URL if self._isPrivate else Constants.WS_PUBLIC_URL self._client: Optional[websockets.WebSocketClientProtocol] = None From a1529f31064528b759b2b14c5f03449ce77c1ac9 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Sun, 7 Mar 2021 12:04:59 +0000 Subject: [PATCH 10/33] HitBTC: Fix some imports and constants --- .../hitbtc_api_user_stream_data_source.py | 3 ++- .../exchange/hitbtc/hitbtc_constants.py | 2 +- .../exchange/hitbtc/hitbtc_exchange.py | 27 ++++++++++--------- .../exchange/hitbtc/hitbtc_order_book.py | 6 ++--- .../hitbtc/hitbtc_order_book_tracker.py | 4 +-- .../hitbtc/hitbtc_user_stream_tracker.py | 4 +-- .../connector/exchange/hitbtc/hitbtc_utils.py | 7 +++-- 7 files changed, 27 insertions(+), 26 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py index 829ee25940..63d0de5076 100755 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py @@ -70,6 +70,7 @@ async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: a raise except Exception: self.logger().error( - "Unexpected error with HitBTC WebSocket connection. " "Retrying after 30 seconds...", exc_info=True + f"Unexpected error with {Constants.EXCHANGE_NAME} WebSocket connection. " + "Retrying after 30 seconds...", exc_info=True ) await asyncio.sleep(30.0) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py index 1dd8fdcc20..b67808b2be 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py @@ -1,6 +1,6 @@ # A single source of truth for constant variables related to the exchange class Constants: - EXCHANGE_NAME = "hitbtc" + EXCHANGE_NAME = "HitBTC" REST_URL = "https://api.hitbtc.com/api/2" # WS_PRIVATE_URL = "wss://stream.crypto.com/v2/user" WS_PRIVATE_URL = "wss://api.hitbtc.com/api/2/ws/trading" diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index 85cc08c30e..7f61b5db40 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -252,8 +252,8 @@ async def _trading_rules_polling_loop(self): except Exception as e: self.logger().network(f"Unexpected error while fetching trading rules. Error: {str(e)}", exc_info=True, - app_warning_msg="Could not fetch new trading rules from HitBTC. " - "Check network connection.") + app_warning_msg=("Could not fetch new trading rules from " + f"{Constants.EXCHANGE_NAME}. Check network connection.")) await asyncio.sleep(0.5) async def _update_trading_rules(self): @@ -474,10 +474,10 @@ async def _create_order(self, error_reason = str(e.error_payload['error']) self.stop_tracking_order(order_id) self.logger().network( - f"Error submitting {trade_type.name} {order_type.name} order to HitBTC for " + f"Error submitting {trade_type.name} {order_type.name} order to {Constants.EXCHANGE_NAME} for " f"{amount} {trading_pair} {price} - {error_reason}.", exc_info=True, - app_warning_msg=(f"Error submitting order to HitBTC - {error_reason}.") + app_warning_msg=(f"Error submitting order to {Constants.EXCHANGE_NAME} - {error_reason}.") ) self.trigger_event(MarketEvent.OrderFailure, MarketOrderFailureEvent(self.current_timestamp, order_id, order_type)) @@ -534,7 +534,7 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: self.logger().network( f"Failed to cancel order {order_id}: {error_reason}", exc_info=True, - app_warning_msg=f"Failed to cancel the order {order_id} on HitBTC. " + app_warning_msg=f"Failed to cancel the order {order_id} on {Constants.EXCHANGE_NAME}. " f"Check API key and network connection." ) return CancellationResult(order_id, False) @@ -557,10 +557,10 @@ async def _status_polling_loop(self): raise except Exception as e: self.logger().error(str(e), exc_info=True) - self.logger().network("Unexpected error while fetching account updates.", - exc_info=True, - app_warning_msg="Could not fetch account updates from HitBTC. " - "Check API key and network connection.") + warn_msg = (f"Could not fetch account updates from {Constants.EXCHANGE_NAME}. " + "Check API key and network connection.") + self.logger().network("Unexpected error while fetching account updates.", exc_info=True, + app_warning_msg=warn_msg) await asyncio.sleep(0.5) async def _update_balances(self): @@ -749,7 +749,8 @@ async def cancel_all(self, timeout_seconds: float) -> List[CancellationResult]: self.logger().network( "Unexpected error cancelling orders.", exc_info=True, - app_warning_msg="Failed to cancel all orders on HitBTC. Check API key and network connection." + app_warning_msg=(f"Failed to cancel all orders on {Constants.EXCHANGE_NAME}. " + "Check API key and network connection.") ) return cancellation_results @@ -792,9 +793,9 @@ async def _iter_user_event_queue(self) -> AsyncIterable[Dict[str, any]]: raise except Exception: self.logger().network( - "Unknown error. Retrying after 1 seconds.", - exc_info=True, - app_warning_msg="Could not fetch user events from HitBTC. Check API key and network connection." + "Unknown error. Retrying after 1 seconds.", exc_info=True, + app_warning_msg=(f"Could not fetch user events from {Constants.EXCHANGE_NAME}. " + "Check API key and network connection.") ) await asyncio.sleep(1.0) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py index 711beaa961..1a3c91a121 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import logging -import hummingbot.connector.exchange.hitbtc.hitbtc_constants as constants +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants from sqlalchemy.engine import RowProxy from typing import ( @@ -139,8 +139,8 @@ def trade_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None @classmethod def from_snapshot(cls, snapshot: OrderBookMessage): - raise NotImplementedError(constants.EXCHANGE_NAME + " order book needs to retain individual order data.") + raise NotImplementedError(Constants.EXCHANGE_NAME + " order book needs to retain individual order data.") @classmethod def restore_from_snapshot_and_diffs(self, snapshot: OrderBookMessage, diffs: List[OrderBookMessage]): - raise NotImplementedError(constants.EXCHANGE_NAME + " order book needs to retain individual order data.") + raise NotImplementedError(Constants.EXCHANGE_NAME + " order book needs to retain individual order data.") diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py index 19707fe7dc..55bdce0d43 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py @@ -2,7 +2,7 @@ import asyncio import bisect import logging -import hummingbot.connector.exchange.hitbtc.hitbtc_constants as constants +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants import time from collections import defaultdict, deque @@ -46,7 +46,7 @@ def exchange_name(self) -> str: """ Name of the current exchange """ - return constants.EXCHANGE_NAME + return Constants.EXCHANGE_NAME async def _track_single_book(self, trading_pair: str): """ diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_user_stream_tracker.py b/hummingbot/connector/exchange/hitbtc/hitbtc_user_stream_tracker.py index 6e17029209..7b04002ccd 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_user_stream_tracker.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_user_stream_tracker.py @@ -18,7 +18,7 @@ from hummingbot.connector.exchange.hitbtc.hitbtc_api_user_stream_data_source import \ HitbtcAPIUserStreamDataSource from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth -from hummingbot.connector.exchange.hitbtc.hitbtc_constants import EXCHANGE_NAME +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants class HitbtcUserStreamTracker(UserStreamTracker): @@ -60,7 +60,7 @@ def exchange_name(self) -> str: *required Name of the current exchange """ - return EXCHANGE_NAME + return Constants.EXCHANGE_NAME async def start(self): """ diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py index 1e1b67cd9d..8ce4bb999c 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py @@ -12,10 +12,9 @@ ) from hummingbot.core.utils.tracking_nonce import get_tracking_nonce -from .hitbtc_constants import Constants - from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.config_methods import using_exchange +from .hitbtc_constants import Constants TRADING_PAIR_SPLITTER = re.compile(Constants.TRADING_PAIR_SPLITTER) @@ -160,13 +159,13 @@ async def generic_api_request(method, KEYS = { "hitbtc_api_key": ConfigVar(key="hitbtc_api_key", - prompt="Enter your HitBTC API key >>> ", + prompt=f"Enter your {Constants.EXCHANGE_NAME} API key >>> ", required_if=using_exchange("hitbtc"), is_secure=True, is_connect_key=True), "hitbtc_secret_key": ConfigVar(key="hitbtc_secret_key", - prompt="Enter your HitBTC secret key >>> ", + prompt=f"Enter your {Constants.EXCHANGE_NAME} secret key >>> ", required_if=using_exchange("hitbtc"), is_secure=True, is_connect_key=True), From d6ab4468eb61416607608ac5403ba8f07e61e1c4 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Sun, 7 Mar 2021 12:07:23 +0000 Subject: [PATCH 11/33] HitBTC: Fix typo in auth --- hummingbot/connector/exchange/hitbtc/hitbtc_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py b/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py index 5b54054478..b89c76588e 100755 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py @@ -15,7 +15,7 @@ def __init__(self, api_key: str, secret_key: str): self.api_key = api_key self.secret_key = secret_key - def generate_auth_dict( + def generate_auth( self, method: str, url: str, From c6f0dc1d5505e0954d969d5f63b032fbbe09749b Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Sun, 7 Mar 2021 12:23:07 +0000 Subject: [PATCH 12/33] HitBTC: Add template vars --- hummingbot/templates/conf_fee_overrides_TEMPLATE.yml | 3 +++ hummingbot/templates/conf_global_TEMPLATE.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml b/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml index ce38789572..ed08e2fa90 100644 --- a/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml +++ b/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml @@ -45,6 +45,9 @@ dolomite_maker_fee_amount: dolomite_taker_fee_amount: +hitbtc_maker_fee: +hitbtc_taker_fee: + loopring_maker_fee: loopring_taker_fee: diff --git a/hummingbot/templates/conf_global_TEMPLATE.yml b/hummingbot/templates/conf_global_TEMPLATE.yml index e66886f105..2d3af846f3 100644 --- a/hummingbot/templates/conf_global_TEMPLATE.yml +++ b/hummingbot/templates/conf_global_TEMPLATE.yml @@ -59,6 +59,9 @@ crypto_com_api_key: null crypto_com_secret_key: null +hitbtc_api_key: null +hitbtc_secret_key: null + bitfinex_api_key: null bitfinex_secret_key: null From 924e58393c339ae368b1f05f56f17d048086559c Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Sun, 7 Mar 2021 13:06:03 +0000 Subject: [PATCH 13/33] HitBTC: Fix auth and balances endpoint --- .../connector/exchange/hitbtc/hitbtc_auth.py | 20 ++++++++++--------- .../exchange/hitbtc/hitbtc_constants.py | 3 ++- .../exchange/hitbtc/hitbtc_exchange.py | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py b/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py index b89c76588e..be667307b7 100755 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py @@ -28,16 +28,16 @@ def generate_auth( nonce = str(int(time.time())) full_url = f"{url}" body = "" - if len(params) > 0 and method.upper() == "GET": + if params is not None and len(params) > 0 and method.upper() == "GET": query_string = "&".join([f"{k}={v}" for k, v in params.items()]) full_url = f"{url}?{query_string}" - elif len(params) > 0 and method.upper() == "POST": + elif params is not None and len(params) > 0 and method.upper() == "POST": body = ujson.dumps(params) - payload = f"{method}{nonce}{full_url}{body}" + payload = f"{method.upper()}{nonce}{full_url}{body}" sig = hmac.new( - self.secret_key.encode('utf-8'), - payload.encode('utf-8'), + self.secret_key.encode(), + payload.encode(), hashlib.sha256 ).hexdigest() @@ -66,8 +66,10 @@ def get_headers(self, :return: a dictionary of auth headers """ nonce, sig = self.generate_auth(method, url, params) - payload = b64encode(f"{self.api_key}:{nonce}:{sig}".encode('utf-8')).decode().strip() - return { - "Authorization": f"HS256 {payload}", - "Content-Type": 'application/json', + payload = b64encode(f"{self.api_key}:{nonce}:{sig}".encode()).decode().strip() + headers = { + "Authorization": f"HS256 {payload}" } + if params is not None and len(params) > 0: + headers["Content-Type"] = "application/json" + return headers diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py index b67808b2be..3f5b717be4 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py @@ -2,6 +2,7 @@ class Constants: EXCHANGE_NAME = "HitBTC" REST_URL = "https://api.hitbtc.com/api/2" + REST_URL_AUTH = "/api/2" # WS_PRIVATE_URL = "wss://stream.crypto.com/v2/user" WS_PRIVATE_URL = "wss://api.hitbtc.com/api/2/ws/trading" # WS_PUBLIC_URL = "wss://stream.crypto.com/v2/market" @@ -19,7 +20,7 @@ class Constants: "ORDER_DELETE": "order/{id}", "ORDER_STATUS": "order/{id}", "USER_ORDERS": "order", - "USER_BALANCES": "account/balance", + "USER_BALANCES": "trading/balance", } WS_SUB = { diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index 7f61b5db40..161bdf9dfa 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -314,7 +314,7 @@ async def _api_request(self, url = f"{Constants.REST_URL}/{endpoint}" shared_client = await self._http_client() if is_auth_required: - headers = self._hitbtc_auth.get_headers(method, url, params) + headers = self._hitbtc_auth.get_headers(method, f"{Constants.REST_URL_AUTH}/{endpoint}", params) else: headers = {"Content-Type": "application/json"} response_coro = shared_client.request( From 57e82bf45bcdb9e7835904341d02029859806bd3 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Sun, 7 Mar 2021 19:08:39 +0000 Subject: [PATCH 14/33] HitBTC: Fix a bunch of Typos HitBTC: Fix more typos HitBTC: Typos and bug fixes Fixes HitBTC: General housekeeping --- .../hitbtc_api_order_book_data_source.py | 53 ++++++------- .../hitbtc_api_user_stream_data_source.py | 15 +++- .../connector/exchange/hitbtc/hitbtc_auth.py | 59 ++++++++------- .../exchange/hitbtc/hitbtc_constants.py | 52 ++----------- .../exchange/hitbtc/hitbtc_exchange.py | 74 +++++++------------ .../connector/exchange/hitbtc/hitbtc_utils.py | 51 +++++++------ .../exchange/hitbtc/hitbtc_websocket.py | 23 +++--- 7 files changed, 137 insertions(+), 190 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py index 3f2369313a..762db1c8e5 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py @@ -18,7 +18,7 @@ str_date_to_ts, convert_to_exchange_trading_pair, convert_from_exchange_trading_pair, - generic_api_request, + api_call_with_retries, HitbtcAPIError, ) @@ -41,22 +41,24 @@ def __init__(self, trading_pairs: List[str] = None): async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, Decimal]: results = {} if len(trading_pairs) > 1: - tickers = await generic_api_request("get", Constants.ENDPOINT["TICKER"]) + tickers: List[Dict[Any]] = await api_call_with_retries("GET", Constants.ENDPOINT["TICKER"]) for trading_pair in trading_pairs: - ex_pair = convert_to_exchange_trading_pair(trading_pair) + ex_pair: str = convert_to_exchange_trading_pair(trading_pair) if len(trading_pairs) > 1: - ticker = list([tic for tic in tickers if tic['symbol'] == ex_pair])[0] + ticker: Dict[Any] = list([tic for tic in tickers if tic['symbol'] == ex_pair])[0] else: url_endpoint = Constants.ENDPOINT["TICKER_SINGLE"].format(trading_pair=ex_pair) - ticker = await generic_api_request("get", url_endpoint) - results[trading_pair] = Decimal(str(ticker["last"])) + ticker: Dict[Any] = await api_call_with_retries("GET", url_endpoint) + results[trading_pair]: Decimal = Decimal(str(ticker["last"])) return results @staticmethod async def fetch_trading_pairs() -> List[str]: try: - symbols: List[Dict[str, Any]] = await generic_api_request("get", Constants.ENDPOINT["SYMBOL"]) - return [convert_from_exchange_trading_pair(sym["id"]) for sym in symbols] + symbols: List[Dict[str, Any]] = await api_call_with_retries("GET", Constants.ENDPOINT["SYMBOL"]) + trading_pairs: List[str] = list([convert_from_exchange_trading_pair(sym["id"]) for sym in symbols]) + # Filter out unmatched pairs so nothing breaks + return [sym for sym in trading_pairs if sym is not None] except Exception: # Do nothing if the request fails -- there will be no autocomplete for huobi trading pairs pass @@ -69,19 +71,13 @@ async def get_order_book_data(trading_pair: str) -> Dict[str, any]: """ try: ex_pair = convert_to_exchange_trading_pair(trading_pair) - orderbook_response = await generic_api_request("get", - Constants.ENDPOINT["ORDER_BOOK"], - params={ - "limit": 150, - "symbols": ex_pair - }) - orderbook_data = orderbook_response[ex_pair] - return orderbook_data + orderbook_response: Dict[Any] = await api_call_with_retries("GET", Constants.ENDPOINT["ORDER_BOOK"], + params={"limit": 150, "symbols": ex_pair}) + return orderbook_response[ex_pair] except HitbtcAPIError as e: raise IOError( f"Error fetching OrderBook for {trading_pair} at {Constants.EXCHANGE_NAME}. " - f"HTTP status is {e.error_payload['status']}. Error is {e.error_payload['error']}." - ) + f"HTTP status is {e.error_payload['status']}. Error is {e.error_payload['error']}.") async def get_new_order_book(self, trading_pair: str) -> OrderBook: snapshot: Dict[str, Any] = await self.get_order_book_data(trading_pair) @@ -89,8 +85,7 @@ async def get_new_order_book(self, trading_pair: str) -> OrderBook: snapshot_msg: OrderBookMessage = HitbtcOrderBook.snapshot_message_from_exchange( snapshot, snapshot_timestamp, - metadata={"trading_pair": trading_pair} - ) + metadata={"trading_pair": trading_pair}) order_book = self.order_book_create_function() active_order_tracker: HitbtcActiveOrderTracker = HitbtcActiveOrderTracker() bids, asks = active_order_tracker.convert_snapshot_message_to_order_book_row(snapshot_msg) @@ -124,8 +119,7 @@ async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asynci trade_msg: OrderBookMessage = HitbtcOrderBook.trade_message_from_exchange( trade, trade_timestamp, - metadata={"trading_pair": pair} - ) + metadata={"trading_pair": pair}) output.put_nowait(trade_msg) except asyncio.CancelledError: @@ -170,19 +164,16 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp orderbook_msg: OrderBookMessage = order_book_msg_cls( order_book_data, timestamp, - metadata={"trading_pair": pair} - ) + metadata={"trading_pair": pair}) output.put_nowait(orderbook_msg) except asyncio.CancelledError: raise except Exception: self.logger().network( - "Unexpected error with WebSocket connection.", - exc_info=True, + "Unexpected error with WebSocket connection.", exc_info=True, app_warning_msg="Unexpected error with WebSocket connection. Retrying in 30 seconds. " - "Check network connection." - ) + "Check network connection.") await asyncio.sleep(30.0) finally: await ws.disconnect() @@ -210,11 +201,9 @@ async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, raise except Exception: self.logger().network( - "Unexpected error with WebSocket connection.", - exc_info=True, + "Unexpected error with WebSocket connection.", exc_info=True, app_warning_msg="Unexpected error with WebSocket connection. Retrying in 5 seconds. " - "Check network connection." - ) + "Check network connection.") await asyncio.sleep(5.0) this_hour: pd.Timestamp = pd.Timestamp.utcnow().replace(minute=0, second=0, microsecond=0) next_hour: pd.Timestamp = this_hour + pd.Timedelta(hours=1) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py index 63d0de5076..5cc2bd03ad 100755 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py @@ -3,7 +3,12 @@ import time import asyncio import logging -from typing import Optional, List, AsyncIterable, Any +from typing import ( + Any, + AsyncIterable, + List, + Optional, +) from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource from hummingbot.logger import HummingbotLogger from .hitbtc_constants import Constants @@ -40,12 +45,15 @@ async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: try: ws = HitbtcWebsocket(self._hitbtc_auth) + await ws.connect() + await ws.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"]) + async for msg in ws.on_message(): - # print(f"WS_SOCKET: {msg}") yield msg self._last_recv_time = time.time() + if (msg.get("result") is None): continue except Exception as e: @@ -71,6 +79,5 @@ async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: a except Exception: self.logger().error( f"Unexpected error with {Constants.EXCHANGE_NAME} WebSocket connection. " - "Retrying after 30 seconds...", exc_info=True - ) + "Retrying after 30 seconds...", exc_info=True) await asyncio.sleep(30.0) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py b/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py index be667307b7..58dd25e4ac 100755 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py @@ -1,7 +1,6 @@ import hmac import hashlib import time -import ujson from base64 import b64encode from typing import Dict, Any @@ -15,61 +14,61 @@ def __init__(self, api_key: str, secret_key: str): self.api_key = api_key self.secret_key = secret_key - def generate_auth( + def generate_payload( self, method: str, url: str, - params: Dict[str, Any] = None + params: Dict[str, Any] = None, + data: str = None, ): """ - Generates authentication signature and return it with the nonce used - :return: a tuple of the nonce used and the request signature + Generates authentication payload and returns it. + :return: A base64 encoded payload for the authentication header. """ + # Nonce is standard EPOCH timestamp only accurate to 1s nonce = str(int(time.time())) - full_url = f"{url}" body = "" - if params is not None and len(params) > 0 and method.upper() == "GET": + if method == "GET" and params is not None and len(params) > 0: + # Need to build the full URL with query string for HS256 sig query_string = "&".join([f"{k}={v}" for k, v in params.items()]) - full_url = f"{url}?{query_string}" - elif params is not None and len(params) > 0 and method.upper() == "POST": - body = ujson.dumps(params) - payload = f"{method.upper()}{nonce}{full_url}{body}" - - sig = hmac.new( - self.secret_key.encode(), - payload.encode(), - hashlib.sha256 - ).hexdigest() - - return (nonce, sig) + url = f"{url}?{query_string}" + elif method == "POST" and data is not None and len(data) > 0: + body = data + # Concat payload + payload = f"{method}{nonce}{url}{body}" + # Create HS256 sig + sig = hmac.new(self.secret_key.encode(), payload.encode(), hashlib.sha256).hexdigest() + # Base64 encode it with public key and nonce + return b64encode(f"{self.api_key}:{nonce}:{sig}".encode()).decode().strip() def generate_auth_dict_ws(self, nonce: int): - data = { + """ + Generates an authentication params for HitBTC websockets login + :return: a dictionary of auth params + """ + return { "algo": "HS256", "pKey": self.api_key, "nonce": nonce, + "signature": hmac.new(self.secret_key.encode('utf-8'), + str(nonce).encode('utf-8'), + hashlib.sha256).hexdigest() } - data['signature'] = hmac.new( - self.secret_key.encode('utf-8'), - str(nonce).encode('utf-8'), - hashlib.sha256 - ).hexdigest() - return data def get_headers(self, method, url, - params) -> Dict[str, Any]: + params, + data) -> Dict[str, Any]: """ Generates authentication headers required by HitBTC :return: a dictionary of auth headers """ - nonce, sig = self.generate_auth(method, url, params) - payload = b64encode(f"{self.api_key}:{nonce}:{sig}".encode()).decode().strip() + payload = self.generate_payload(method, url, params, data) headers = { "Authorization": f"HS256 {payload}" } - if params is not None and len(params) > 0: + if data is not None and len(data) > 0: headers["Content-Type"] = "application/json" return headers diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py index 3f5b717be4..3ac2200f7c 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py @@ -1,6 +1,6 @@ # A single source of truth for constant variables related to the exchange class Constants: - EXCHANGE_NAME = "HitBTC" + EXCHANGE_NAME = "hitbtc" REST_URL = "https://api.hitbtc.com/api/2" REST_URL_AUTH = "/api/2" # WS_PRIVATE_URL = "wss://stream.crypto.com/v2/user" @@ -31,59 +31,19 @@ class Constants: } WS_METHODS = { - "ORDER_SNAPSHOT": "snapshotOrderbook", - "ORDER_UPDATE": "updateOrderbook", + "ORDERS_SNAPSHOT": "snapshotOrderbook", + "ORDERS_UPDATE": "updateOrderbook", "TRADES_SNAPSHOT": "snapshotTrades", "TRADES_UPDATE": "updateTrades", "USER_ORDERS": "activeOrders", "USER_TRADES": "report", } - API_REASONS = { - 0: "Success", - 403: "Action is forbidden for account", # HTTP: 401 - 429: "Too many requests", # HTTP: 429 - 500: "Internal Server Error", # HTTP: 500 - 503: "Service Unavailable", # HTTP: 503 - 504: "Gateway Timeout", # HTTP: 504 - 1001: "Authorization required", # HTTP: 401 - 1002: "Authorization required or has been failed", # HTTP: 401 - 1003: "Action forbidden for this API key", # HTTP: 403 - 1004: "Unsupported authorization method", # HTTP: 401 - 2001: "Symbol not found", # HTTP: 400 - 2002: "Currency not found", # HTTP: 400 - 2010: "Quantity not a valid number", # HTTP: 400 - 2011: "Quantity too low", # HTTP: 400 - 2012: "Bad quantity", # HTTP: 400 - 2020: "Price not a valid number", # HTTP: 400 - 2021: "Price too low", # HTTP: 400 - 2022: "Bad price", # HTTP: 400 - 20001: "Insufficient funds", # HTTP: 400 - 20002: "Order not found", # HTTP: 400 - 20003: "Limit exceeded", # HTTP: 400 - 20004: "Transaction not found", # HTTP: 400 - 20005: "Payout not found", # HTTP: 400 - 20006: "Payout already committed", # HTTP: 400 - 20007: "Payout already rolled back", # HTTP: 400 - 20008: "Duplicate clientOrderId", # HTTP: 400 - 20009: "Price and quantity not changed", # HTTP: 400 - 20010: "Exchange temporary closed", # HTTP: 400 - 20011: "Payout address is invalid", # HTTP: 400 - 20014: "Offchain for this payout is unavailable", # HTTP: 400 - 20032: "Margin account or position not found", # HTTP: 400 - 20033: "Position not changed", # HTTP: 400 - 20034: "Position in close only state", # HTTP: 400 - 20040: "Margin trading forbidden", # HTTP: 400 - 20080: "Internal order execution deadline exceeded", # HTTP: 400. - 10001: "Validation error", # HTTP: 400 - 10021: "User disabled", # HTTP: 400 - - } - # Timeouts MESSAGE_TIMEOUT = 30.0 PING_TIMEOUT = 10.0 API_CALL_TIMEOUT = 10.0 + API_MAX_RETRIES = 4 # Intervals # Only used when nothing is received from WS @@ -92,6 +52,8 @@ class Constants: LONG_POLL_INTERVAL = 20.0 # One minute should be fine for order status since we get these via WS UPDATE_ORDER_STATUS_INTERVAL = 60.0 + # 10 minute interval to update trading rules, these would likely never change whilst running. + INTERVAL_TRADING_RULES = 600 # Trading pair splitter regex - TRADING_PAIR_SPLITTER = r"^(\w+)(BTC|BCH|DAI|DDRST|EOSDT|EOS|ETH|EURS|IDRT|PAX|BUSD|GUSD|TUSD|USDC|USD)$" + TRADING_PAIR_SPLITTER = r"^(\w+)(BTC|BCH|DAI|DDRST|EOSDT|EOS|ETH|EURS|IDRT|PAX|BUSD|GUSD|TUSD|USDC|USDT|USD)$" diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index 161bdf9dfa..1986a1354c 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -11,6 +11,7 @@ import aiohttp import math import time +import ujson from async_timeout import timeout from hummingbot.core.network_iterator import NetworkStatus @@ -43,6 +44,7 @@ convert_from_exchange_trading_pair, convert_to_exchange_trading_pair, get_new_client_order_id, + aiohttp_response_with_errors, retry_sleep_time, str_date_to_ts, HitbtcAPIError, @@ -222,7 +224,7 @@ async def check_network(self) -> NetworkStatus: """ try: # since there is no ping endpoint, the lowest rate call is to get BTC-USD symbol - await self._api_request("get", + await self._api_request("GET", Constants.ENDPOINT['SYMBOL'], params={'symbols': 'BTCUSD'}) except asyncio.CancelledError: @@ -257,7 +259,7 @@ async def _trading_rules_polling_loop(self): await asyncio.sleep(0.5) async def _update_trading_rules(self): - symbols_info = await self._api_request("get", endpoint=Constants.ENDPOINT['SYMBOL']) + symbols_info = await self._api_request("GET", endpoint=Constants.ENDPOINT['SYMBOL']) self._trading_rules.clear() self._trading_rules = self._format_trading_rules(symbols_info) @@ -313,33 +315,20 @@ async def _api_request(self, """ url = f"{Constants.REST_URL}/{endpoint}" shared_client = await self._http_client() + # Turn `params` into either GET params or POST body data + qs_params: dict = params if method.upper() == "GET" else None + req_data: str = ujson.dumps(params) if (method.upper() == "POST" and params is not None) else None + # Generate auth headers if needed. + headers: dict = {"Content-Type": "application/json"} if is_auth_required: - headers = self._hitbtc_auth.get_headers(method, f"{Constants.REST_URL_AUTH}/{endpoint}", params) - else: - headers = {"Content-Type": "application/json"} - response_coro = shared_client.request( - method=method.upper(), url=url, headers=headers, params=params, timeout=Constants.API_CALL_TIMEOUT - ) - http_status, parsed_response, request_errors = None, None, False - try: - async with response_coro as response: - http_status = response.status - try: - parsed_response = await response.json() - except Exception: - request_errors = True - try: - parsed_response = str(await response.read()) - if len(parsed_response) > 100: - parsed_response = f"{parsed_response[:100]} ... (truncated)" - except Exception: - pass - if response.status not in [200, 201] or parsed_response is None: - request_errors = True - except Exception: - request_errors = True + headers: dict = self._hitbtc_auth.get_headers(method, f"{Constants.REST_URL_AUTH}/{endpoint}", + qs_params, req_data) + # Build request coro + response_coro = shared_client.request(method=method.upper(), url=url, headers=headers, + params=qs_params, data=req_data, timeout=Constants.API_CALL_TIMEOUT) + http_status, parsed_response, request_errors = await aiohttp_response_with_errors(response_coro) if request_errors or parsed_response is None: - if try_count < 4: + if try_count < Constants.API_MAX_RETRIES: try_count += 1 time_sleep = retry_sleep_time(try_count) self.logger().info(f"Error fetching data from {url}. HTTP status is {http_status}. " @@ -348,8 +337,6 @@ async def _api_request(self, return await self._api_request(method=method, endpoint=endpoint, params=params, is_auth_required=is_auth_required, try_count=try_count) else: - self.logger().network(f"Error fetching data from {url}. HTTP status is {http_status}. " - f"Final msg: {parsed_response}.") raise HitbtcAPIError({"error": parsed_response, "status": http_status}) if "error" in parsed_response: raise HitbtcAPIError(parsed_response) @@ -453,7 +440,7 @@ async def _create_order(self, api_params["postOnly"] = "true" self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, order_type) try: - order_result = await self._api_request("post", Constants.ENDPOINT["ORDER_CREATE"], api_params, True) + order_result = await self._api_request("POST", Constants.ENDPOINT["ORDER_CREATE"], api_params, True) exchange_order_id = str(order_result["id"]) tracked_order = self._in_flight_orders.get(order_id) if tracked_order is not None: @@ -514,7 +501,7 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: """ Executes order cancellation process by first calling cancel-order API. The API result doesn't confirm whether the cancellation is successful, it simply states it receives the request. - :param trading_pair: The market trading pair + :param trading_pair: The market trading pair (Unused during cancel on HitBTC) :param order_id: The internal order id order.last_state to change to CANCELED """ @@ -525,7 +512,7 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: if tracked_order.exchange_order_id is None: await tracked_order.get_exchange_order_id() # ex_order_id = tracked_order.exchange_order_id - await self._api_request("delete", Constants.ENDPOINT["ORDER_DELETE"].format(id=order_id), True) + await self._api_request("DELETE", Constants.ENDPOINT["ORDER_DELETE"].format(id=order_id), True) return CancellationResult(order_id, True) except asyncio.CancelledError: raise @@ -569,7 +556,7 @@ async def _update_balances(self): """ local_asset_names = set(self._account_balances.keys()) remote_asset_names = set() - account_info = await self._api_request("get", Constants.ENDPOINT["USER_BALANCES"], is_auth_required=True) + account_info = await self._api_request("GET", Constants.ENDPOINT["USER_BALANCES"], is_auth_required=True) for account in account_info: asset_name = account["currency"] self._account_available_balances[asset_name] = Decimal(str(account["available"])) @@ -594,7 +581,7 @@ async def _update_order_status(self): for tracked_order in tracked_orders: # exchange_order_id = await tracked_order.get_exchange_order_id() order_id = tracked_order.client_order_id - tasks.append(self._api_request("get", + tasks.append(self._api_request("GET", Constants.ENDPOINT["ORDER_STATUS"].format(id=order_id), is_auth_required=True)) self.logger().debug(f"Polling for order status updates of {len(tasks)} orders.") @@ -642,19 +629,14 @@ def _process_order_message(self, order_msg: Dict[str, Any]): if tracked_order.is_cancelled: self.logger().info(f"Successfully cancelled order {client_order_id}.") self.trigger_event(MarketEvent.OrderCancelled, - OrderCancelledEvent( - self.current_timestamp, - client_order_id)) + OrderCancelledEvent(self.current_timestamp, client_order_id)) tracked_order.cancelled_event.set() self.stop_tracking_order(client_order_id) elif tracked_order.is_failure: self.logger().info(f"The market order {client_order_id} has failed according to order status API. ") self.trigger_event(MarketEvent.OrderFailure, MarketOrderFailureEvent( - self.current_timestamp, - client_order_id, - tracked_order.order_type - )) + self.current_timestamp, client_order_id, tracked_order.order_type)) self.stop_tracking_order(client_order_id) async def _process_trade_message(self, trade_msg: Dict[str, Any]): @@ -740,15 +722,14 @@ async def cancel_all(self, timeout_seconds: float) -> List[CancellationResult]: open_orders = [o for o in self._in_flight_orders.values() if not o.is_done] if len(open_orders) == 0: return [] - tasks = [self._execute_cancel(o.client_order_id) for o in open_orders] + tasks = [self._execute_cancel(o.trading_pair, o.client_order_id) for o in open_orders] cancellation_results = [] try: async with timeout(timeout_seconds): cancellation_results = await safe_gather(*tasks, return_exceptions=True) except Exception: self.logger().network( - "Unexpected error cancelling orders.", - exc_info=True, + "Unexpected error cancelling orders.", exc_info=True, app_warning_msg=(f"Failed to cancel all orders on {Constants.EXCHANGE_NAME}. " "Check API key and network connection.") ) @@ -795,8 +776,7 @@ async def _iter_user_event_queue(self) -> AsyncIterable[Dict[str, any]]: self.logger().network( "Unknown error. Retrying after 1 seconds.", exc_info=True, app_warning_msg=(f"Could not fetch user events from {Constants.EXCHANGE_NAME}. " - "Check API key and network connection.") - ) + "Check API key and network connection.")) await asyncio.sleep(1.0) async def _user_stream_event_listener(self): @@ -828,7 +808,7 @@ async def _user_stream_event_listener(self): # This is currently unused, but looks like a future addition. async def get_open_orders(self) -> List[OpenOrder]: - result = await self._api_request("get", Constants.ENDPOINT["USER_ORDERS"], is_auth_required=True) + result = await self._api_request("GET", Constants.ENDPOINT["USER_ORDERS"], is_auth_required=True) ret_val = [] for order in result: if Constants.HBOT_BROKER_ID not in order["clientOrderId"]: diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py index 8ce4bb999c..d48125020b 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py @@ -77,16 +77,17 @@ def split_trading_pair(trading_pair: str) -> Optional[Tuple[str, str]]: def convert_from_exchange_trading_pair(ex_trading_pair: str) -> Optional[str]: - if split_trading_pair(ex_trading_pair) is None: + regex_match = split_trading_pair(ex_trading_pair) + if regex_match is None: return None - # Altmarkets uses lowercase (btcusdt) + # HitBTC uses uppercase (BTCUSDT) base_asset, quote_asset = split_trading_pair(ex_trading_pair) return f"{base_asset.upper()}-{quote_asset.upper()}" def convert_to_exchange_trading_pair(hb_trading_pair: str) -> str: - # Altmarkets uses lowercase (btcusdt) - return hb_trading_pair.replace("-", "").lower() + # HitBTC uses uppercase (BTCUSDT) + return hb_trading_pair.replace("-", "").upper() def get_new_client_order_id(is_buy: bool, trading_pair: str) -> str: @@ -109,20 +110,10 @@ def retry_sleep_time(try_count: int) -> float: return float(2 + float(randSleep * (1 + (try_count ** try_count)))) -async def generic_api_request(method, - endpoint, - params: Optional[Dict[str, Any]] = None, - shared_client=None, - try_count: int = 0) -> Dict[str, Any]: - url = f"{Constants.REST_URL}/{endpoint}" - headers = {"Content-Type": "application/json"} - http_client = shared_client if shared_client is not None else aiohttp.ClientSession() - response_coro = http_client.request( - method=method.upper(), url=url, headers=headers, params=params, timeout=Constants.API_CALL_TIMEOUT - ) +async def aiohttp_response_with_errors(request_coroutine): http_status, parsed_response, request_errors = None, None, False try: - async with response_coro as response: + async with request_coroutine as response: http_status = response.status try: parsed_response = await response.json() @@ -134,24 +125,40 @@ async def generic_api_request(method, parsed_response = f"{parsed_response[:100]} ... (truncated)" except Exception: pass - if response.status not in [200, 201] or parsed_response is None: + TempFailure = (parsed_response is None or + (response.status not in [200, 201] and "error" not in parsed_response)) + if TempFailure: + parsed_response = response.reason if parsed_response is None else parsed_response request_errors = True except Exception: request_errors = True + return http_status, parsed_response, request_errors + + +async def api_call_with_retries(method, + endpoint, + params: Optional[Dict[str, Any]] = None, + shared_client=None, + try_count: int = 0) -> Dict[str, Any]: + url = f"{Constants.REST_URL}/{endpoint}" + headers = {"Content-Type": "application/json"} + http_client = shared_client if shared_client is not None else aiohttp.ClientSession() + # Build request coro + response_coro = http_client.request(method=method.upper(), url=url, headers=headers, + params=params, timeout=Constants.API_CALL_TIMEOUT) + http_status, parsed_response, request_errors = await aiohttp_response_with_errors(response_coro) if shared_client is None: await http_client.close() if request_errors or parsed_response is None: - if try_count < 4: + if try_count < Constants.API_MAX_RETRIES: try_count += 1 time_sleep = retry_sleep_time(try_count) print(f"Error fetching data from {url}. HTTP status is {http_status}. " f"Retrying in {time_sleep:.0f}s.") await asyncio.sleep(time_sleep) - return await generic_api_request(method=method, endpoint=endpoint, params=params, - shared_client=shared_client, try_count=try_count) + return await api_call_with_retries(method=method, endpoint=endpoint, params=params, + shared_client=shared_client, try_count=try_count) else: - print(f"Error fetching data from {url}. HTTP status is {http_status}. " - f"Final msg: {parsed_response}.") raise HitbtcAPIError({"error": parsed_response, "status": http_status}) return parsed_response diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py index 2f72c260eb..3d823fbdea 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py @@ -8,9 +8,10 @@ from typing import ( - Optional, - AsyncIterable, Any, + AsyncIterable, + Dict, + Optional, ) from websockets.exceptions import ConnectionClosed from hummingbot.logger import HummingbotLogger @@ -84,7 +85,7 @@ async def _messages(self) -> AsyncIterable[Any]: await self.disconnect() # emit messages - async def _emit(self, method: str, data: Optional[Any] = {}) -> int: + async def _emit(self, method: str, data: Optional[Dict[str, Any]] = {}) -> int: id = self.generate_request_id() payload = { @@ -98,23 +99,25 @@ async def _emit(self, method: str, data: Optional[Any] = {}) -> int: return id # request via websocket - async def request(self, method: str, data: Optional[Any] = {}) -> int: + async def request(self, method: str, data: Optional[Dict[str, Any]] = {}) -> int: return await self._emit(method, data) # subscribe to a method async def subscribe(self, channel: str, - trading_pair: str, - params: Optional[Any] = {}) -> int: - params['symbol'] = trading_pair + trading_pair: Optional[str] = None, + params: Optional[Dict[str, Any]] = {}) -> int: + if trading_pair is not None: + params['symbol'] = trading_pair return await self.request(f"subscribe{channel}", params) # unsubscribe to a method async def unsubscribe(self, channel: str, - trading_pair: str, - params: Optional[Any] = {}) -> int: - params['symbol'] = trading_pair + trading_pair: Optional[str] = None, + params: Optional[Dict[str, Any]] = {}) -> int: + if trading_pair is not None: + params['symbol'] = trading_pair return await self.request(f"unsubscribe{channel}", params) # listen to messages by method From 2cf472fe5d116c72da4320a2fd9ec92419de3da7 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Mon, 8 Mar 2021 04:19:20 +0000 Subject: [PATCH 15/33] HitBTC: Fix post/delete requests, order cancelling --- .../connector/exchange/hitbtc/hitbtc_auth.py | 22 +++++++++---------- .../exchange/hitbtc/hitbtc_exchange.py | 14 +++++++----- .../exchange/hitbtc/hitbtc_websocket.py | 3 ++- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py b/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py index 58dd25e4ac..ce527ecf72 100755 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py @@ -19,7 +19,6 @@ def generate_payload( method: str, url: str, params: Dict[str, Any] = None, - data: str = None, ): """ Generates authentication payload and returns it. @@ -28,12 +27,13 @@ def generate_payload( # Nonce is standard EPOCH timestamp only accurate to 1s nonce = str(int(time.time())) body = "" - if method == "GET" and params is not None and len(params) > 0: - # Need to build the full URL with query string for HS256 sig + # Need to build the full URL with query string for HS256 sig + if params is not None and len(params) > 0: query_string = "&".join([f"{k}={v}" for k, v in params.items()]) - url = f"{url}?{query_string}" - elif method == "POST" and data is not None and len(data) > 0: - body = data + if method == "GET": + url = f"{url}?{query_string}" + else: + body = query_string # Concat payload payload = f"{method}{nonce}{url}{body}" # Create HS256 sig @@ -59,16 +59,14 @@ def generate_auth_dict_ws(self, def get_headers(self, method, url, - params, - data) -> Dict[str, Any]: + params) -> Dict[str, Any]: """ Generates authentication headers required by HitBTC :return: a dictionary of auth headers """ - payload = self.generate_payload(method, url, params, data) + payload = self.generate_payload(method, url, params) headers = { - "Authorization": f"HS256 {payload}" + "Authorization": f"HS256 {payload}", + "Content-Type": "application/x-www-form-urlencoded", } - if data is not None and len(data) > 0: - headers["Content-Type"] = "application/json" return headers diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index 1986a1354c..f61709697d 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -11,7 +11,6 @@ import aiohttp import math import time -import ujson from async_timeout import timeout from hummingbot.core.network_iterator import NetworkStatus @@ -317,15 +316,16 @@ async def _api_request(self, shared_client = await self._http_client() # Turn `params` into either GET params or POST body data qs_params: dict = params if method.upper() == "GET" else None - req_data: str = ujson.dumps(params) if (method.upper() == "POST" and params is not None) else None + req_form = aiohttp.FormData(params) if method.upper() == "POST" and params is not None else None # Generate auth headers if needed. - headers: dict = {"Content-Type": "application/json"} + headers: dict = {"Content-Type": "application/x-www-form-urlencoded"} if is_auth_required: headers: dict = self._hitbtc_auth.get_headers(method, f"{Constants.REST_URL_AUTH}/{endpoint}", - qs_params, req_data) + params) # Build request coro response_coro = shared_client.request(method=method.upper(), url=url, headers=headers, - params=qs_params, data=req_data, timeout=Constants.API_CALL_TIMEOUT) + params=qs_params, data=req_form, + timeout=Constants.API_CALL_TIMEOUT) http_status, parsed_response, request_errors = await aiohttp_response_with_errors(response_coro) if request_errors or parsed_response is None: if try_count < Constants.API_MAX_RETRIES: @@ -512,7 +512,9 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: if tracked_order.exchange_order_id is None: await tracked_order.get_exchange_order_id() # ex_order_id = tracked_order.exchange_order_id - await self._api_request("DELETE", Constants.ENDPOINT["ORDER_DELETE"].format(id=order_id), True) + await self._api_request("DELETE", + Constants.ENDPOINT["ORDER_DELETE"].format(id=order_id), + is_auth_required=True) return CancellationResult(order_id, True) except asyncio.CancelledError: raise diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py index 3d823fbdea..337a8a4c34 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py @@ -3,6 +3,7 @@ import copy import logging import websockets +import json import ujson from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants @@ -68,7 +69,7 @@ async def _messages(self) -> AsyncIterable[Any]: try: raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=Constants.MESSAGE_TIMEOUT) try: - raw_msg = ujson.loads(raw_msg_str) + raw_msg = json.loads(raw_msg_str) # HitBTC doesn't support ping or heartbeat messages. # Can handle them here if that changes - use `safe_ensure_future`. yield raw_msg From c9c83a00249eaa98a9b2fe5b8f12197f362eaf3d Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Mon, 8 Mar 2021 04:32:44 +0000 Subject: [PATCH 16/33] HitBTC: Fix cancelled orders, orders not existing --- .../exchange/hitbtc/hitbtc_exchange.py | 47 +++++++++++++++---- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index f61709697d..703713d1d2 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -59,6 +59,8 @@ class HitbtcExchange(ExchangeBase): HitbtcExchange connects with HitBTC exchange and provides order book pricing, user account tracking and trading functionality. """ + ORDER_NOT_EXIST_CONFIRMATION_COUNT = 3 + ORDER_NOT_EXIST_CANCEL_COUNT = 2 @classmethod def logger(cls) -> HummingbotLogger: @@ -496,6 +498,8 @@ def stop_tracking_order(self, order_id: str): """ if order_id in self._in_flight_orders: del self._in_flight_orders[order_id] + if order_id in self._order_not_found_records: + del self._order_not_found_records[order_id] async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: """ @@ -505,6 +509,7 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: :param order_id: The internal order id order.last_state to change to CANCELED """ + order_was_cancelled = False try: tracked_order = self._in_flight_orders.get(order_id) if tracked_order is None: @@ -515,11 +520,23 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: await self._api_request("DELETE", Constants.ENDPOINT["ORDER_DELETE"].format(id=order_id), is_auth_required=True) - return CancellationResult(order_id, True) + order_was_cancelled = True except asyncio.CancelledError: raise except HitbtcAPIError as e: - error_reason = str(e.error_payload['error']) + err = e.error_payload['error'] + error_reason = err['message'] if 'message' in err else err + self._order_not_found_records[order_id] = self._order_not_found_records.get(order_id, 0) + 1 + if err['code'] == 20002 and \ + self._order_not_found_records[order_id] >= self.ORDER_NOT_EXIST_CANCEL_COUNT: + order_was_cancelled = True + if order_was_cancelled: + self.logger().info(f"Successfully cancelled order {order_id} on {Constants.EXCHANGE_NAME}.") + self.stop_tracking_order(order_id) + self.trigger_event(MarketEvent.OrderCancelled, + OrderCancelledEvent(self.current_timestamp, order_id)) + return CancellationResult(order_id, True) + else: self.logger().network( f"Failed to cancel order {order_id}: {error_reason}", exc_info=True, @@ -588,13 +605,27 @@ async def _update_order_status(self): is_auth_required=True)) self.logger().debug(f"Polling for order status updates of {len(tasks)} orders.") responses = await safe_gather(*tasks, return_exceptions=True) - for response in responses: - if isinstance(response, Exception): - raise response - if "clientOrderId" not in response: + for response, tracked_order in zip(responses, tracked_orders): + client_order_id = tracked_order.client_order_id + if isinstance(response, HitbtcAPIError): + if response.error_payload['error']['code'] == 20002: + self._order_not_found_records[client_order_id] = \ + self._order_not_found_records.get(client_order_id, 0) + 1 + if self._order_not_found_records[client_order_id] < self.ORDER_NOT_EXIST_CONFIRMATION_COUNT: + # Wait until the order not found error have repeated a few times before actually treating + # it as failed. See: https://github.com/CoinAlpha/hummingbot/issues/601 + continue + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent( + self.current_timestamp, client_order_id, tracked_order.order_type)) + self.stop_tracking_order(client_order_id) + else: + continue + elif "clientOrderId" not in response: self.logger().info(f"_update_order_status clientOrderId not in resp: {response}") continue - self._process_order_message(response) + else: + self._process_order_message(response) def _process_order_message(self, order_msg: Dict[str, Any]): """ @@ -728,7 +759,7 @@ async def cancel_all(self, timeout_seconds: float) -> List[CancellationResult]: cancellation_results = [] try: async with timeout(timeout_seconds): - cancellation_results = await safe_gather(*tasks, return_exceptions=True) + cancellation_results = await safe_gather(*tasks, return_exceptions=False) except Exception: self.logger().network( "Unexpected error cancelling orders.", exc_info=True, From eb8f75d605de5d460848578bc62223d968ad9947 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Mon, 8 Mar 2021 19:46:19 +0000 Subject: [PATCH 17/33] HitBTC: Add to setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 6cd0dd9002..083859ad4c 100755 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ def main(): "hummingbot.connector.exchange.eterbase", "hummingbot.connector.exchange.beaxy", "hummingbot.connector.exchange.bitmax", + "hummingbot.connector.exchange.hitbtc", "hummingbot.connector.derivative", "hummingbot.connector.derivative.binance_perpetual", "hummingbot.script", From 95c7c74693371d733d1ed9be2721acefd693902d Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Mon, 8 Mar 2021 21:03:50 +0000 Subject: [PATCH 18/33] HitBTC: Fix User / Trade Websockets --- .../hitbtc_api_user_stream_data_source.py | 4 +-- .../connector/exchange/hitbtc/hitbtc_auth.py | 4 +-- .../exchange/hitbtc/hitbtc_exchange.py | 9 +++--- .../exchange/hitbtc/hitbtc_in_flight_order.py | 21 +++++++++----- .../exchange/hitbtc/hitbtc_websocket.py | 29 +++++++++---------- 5 files changed, 37 insertions(+), 30 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py index 5cc2bd03ad..af67aed70b 100755 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py @@ -48,13 +48,13 @@ async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: await ws.connect() - await ws.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"]) + await ws.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"], None, {}) async for msg in ws.on_message(): yield msg self._last_recv_time = time.time() - if (msg.get("result") is None): + if (msg.get("params") is None): continue except Exception as e: raise e diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py b/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py index ce527ecf72..be37f2e149 100755 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_auth.py @@ -49,8 +49,8 @@ def generate_auth_dict_ws(self, """ return { "algo": "HS256", - "pKey": self.api_key, - "nonce": nonce, + "pKey": str(self.api_key), + "nonce": str(nonce), "signature": hmac.new(self.secret_key.encode('utf-8'), str(nonce).encode('utf-8'), hashlib.sha256).hexdigest() diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index 703713d1d2..70f96fbe3b 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -715,14 +715,15 @@ async def _process_trade_message(self, trade_msg: Dict[str, Any]): tracked_order.trading_pair, tracked_order.trade_type, tracked_order.order_type, - Decimal(str(trade_msg["tradePrice"])), - Decimal(str(trade_msg["tradeQuantity"])), - TradeFee(0.0, [(tracked_order.quote_asset, -Decimal(str(trade_msg["tradeFee"])))]), + Decimal(str(trade_msg.get("tradePrice", "0"))), + Decimal(str(trade_msg.get("tradeQuantity", "0"))), + TradeFee(0.0, [(tracked_order.quote_asset, Decimal(str(trade_msg.get("tradeFee", "0"))))]), exchange_trade_id=trade_msg["id"] ) ) if math.isclose(tracked_order.executed_amount_base, tracked_order.amount) or \ - tracked_order.executed_amount_base >= tracked_order.amount: + tracked_order.executed_amount_base >= tracked_order.amount or \ + tracked_order.is_done: tracked_order.last_state = "FILLED" self.logger().info(f"The {tracked_order.trade_type.name} order " f"{tracked_order.client_order_id} has completed " diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py b/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py index 8654dddfaa..22ce9e18fc 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py @@ -11,6 +11,8 @@ ) from hummingbot.connector.in_flight_order_base import InFlightOrderBase +s_decimal_0 = Decimal(0) + class HitbtcInFlightOrder(InFlightOrderBase): def __init__(self, @@ -99,22 +101,27 @@ def update_with_trade_update(self, trade_update: Dict[str, Any]) -> bool: "createdAt": "2017-10-20T12:20:05.952Z", "updatedAt": "2017-10-20T12:20:38.708Z", "reportType": "trade", + } + ... Trade variables are only included after fills. + { "tradeQuantity": "0.001", "tradePrice": "0.053868", "tradeId": 55051694, "tradeFee": "-0.000000005" } """ - trade_id = trade_update["clientOrderId"] - # trade_update["orderId"] is type int - if str(trade_update["id"]) != self.exchange_order_id or trade_id in self.trade_id_set: + self.executed_amount_base = Decimal(str(trade_update["cumQuantity"])) + if self.executed_amount_base <= s_decimal_0: + # No trades executed yet. + return False + trade_id = trade_update["updatedAt"] + if trade_id in self.trade_id_set: # trade already recorded return False self.trade_id_set.add(trade_id) - self.executed_amount_base = Decimal(str(trade_update["cumQuantity"])) - self.fee_paid += Decimal(str(trade_update["fee"])) - self.executed_amount_quote += (Decimal(str(trade_update["tradePrice"])) * - Decimal(str(trade_update["tradeQuantity"]))) + self.fee_paid += Decimal(str(trade_update.get("tradeFee", "0"))) + self.executed_amount_quote += (Decimal(str(trade_update.get("tradePrice", "0"))) * + Decimal(str(trade_update.get("tradeQuantity", "0")))) if not self.fee_asset: self.fee_asset = self.quote_asset return True diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py index 337a8a4c34..a721fc0320 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py @@ -4,7 +4,6 @@ import logging import websockets import json -import ujson from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants @@ -40,20 +39,20 @@ def __init__(self, auth: Optional[HitbtcAuth] = None): # connect to exchange async def connect(self): - try: - self._client = await websockets.connect(self._WS_URL) + self._client = await websockets.connect(self._WS_URL) - # if auth class was passed into websocket class - # we need to emit authenticated requests - if self._isPrivate: - auth_params = self._auth.generate_auth_dict_ws(self.generate_request_id()) - await self._emit("login", auth_params) - # TODO: wait for response - await asyncio.sleep(1) + # if auth class was passed into websocket class + # we need to emit authenticated requests + if self._isPrivate: + auth_params = self._auth.generate_auth_dict_ws(self.generate_request_id()) + await self._emit("login", auth_params, no_id=True) + raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=Constants.MESSAGE_TIMEOUT) + json_msg = json.loads(raw_msg_str) + if json_msg.get("result") is not True: + err_msg = json_msg.get('error', {}).get('message') + raise Exception(f"Failed to authenticate to websocket - {err_msg}.") - return self._client - except Exception as e: - self.logger().error(f"Websocket error: '{str(e)}'", exc_info=True) + return self._client # disconnect from exchange async def disconnect(self): @@ -86,7 +85,7 @@ async def _messages(self) -> AsyncIterable[Any]: await self.disconnect() # emit messages - async def _emit(self, method: str, data: Optional[Dict[str, Any]] = {}) -> int: + async def _emit(self, method: str, data: Optional[Dict[str, Any]] = {}, no_id: bool = False) -> int: id = self.generate_request_id() payload = { @@ -95,7 +94,7 @@ async def _emit(self, method: str, data: Optional[Dict[str, Any]] = {}) -> int: "params": copy.deepcopy(data), } - await self._client.send(ujson.dumps(payload)) + await self._client.send(json.dumps(payload)) return id From c4465ca0d3878e3c437f1c1c07766488b4bd9c8b Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Tue, 9 Mar 2021 03:43:35 +0000 Subject: [PATCH 19/33] HitBTC: Fix order book not updating HitBTC: Fix order book snapshots --- .../hitbtc/hitbtc_active_order_tracker.pyx | 43 +++++++------------ .../hitbtc_api_order_book_data_source.py | 3 +- .../exchange/hitbtc/hitbtc_exchange.py | 12 +++--- .../hitbtc/hitbtc_order_book_message.py | 9 ++-- .../hitbtc/hitbtc_order_book_tracker.py | 5 +-- 5 files changed, 31 insertions(+), 41 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx index 248206f060..6d4b57b836 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx @@ -70,8 +70,8 @@ cdef class HitbtcActiveOrderTracker: if len(bid_entries) > 0: bids = np.array( [[timestamp, - float(price), - float(amount), + price, + amount, message.update_id] for price, amount in [self.get_rates_and_quantities(entry) for entry in bid_entries]], dtype="float64", @@ -81,8 +81,8 @@ cdef class HitbtcActiveOrderTracker: if len(ask_entries) > 0: asks = np.array( [[timestamp, - float(price), - float(amount), + price, + amount, message.update_id] for price, amount in [self.get_rates_and_quantities(entry) for entry in ask_entries]], dtype="float64", @@ -104,38 +104,25 @@ cdef class HitbtcActiveOrderTracker: timestamp = message.timestamp content = message.content - for snapshot_orders, active_orders in [(content["bid"], self._active_bids), (content["ask"], self.active_asks)]: - for order in snapshot_orders: - price, amount = self.get_rates_and_quantities(order) - - order_dict = { - "order_id": timestamp, - "amount": amount - } - - if price in active_orders: - active_orders[price][timestamp] = order_dict - else: - active_orders[price] = { - timestamp: order_dict - } + for snapshot_orders, active_orders in [(content["bid"], self._active_bids), (content["ask"], self._active_asks)]: + for entry in snapshot_orders: + price, amount = self.get_rates_and_quantities(entry) + active_orders[price] = amount + # Return the sorted snapshot tables. cdef: np.ndarray[np.float64_t, ndim=2] bids = np.array( [[message.timestamp, - price, - sum([order_dict["amount"] - for order_dict in self._active_bids[price].values()]), + float(price), + float(self._active_bids[price]), message.update_id] - for price in sorted(self._active_bids.keys(), reverse=True)], dtype="float64", ndmin=2) + for price in sorted(self._active_bids.keys(), reverse=True)], dtype='float64', ndmin=2) np.ndarray[np.float64_t, ndim=2] asks = np.array( [[message.timestamp, - price, - sum([order_dict["amount"] - for order_dict in self.active_asks[price].values()]), + float(price), + float(self._active_asks[price]), message.update_id] - for price in sorted(self.active_asks.keys(), reverse=True)], dtype="float64", ndmin=2 - ) + for price in sorted(self._active_asks.keys(), reverse=True)], dtype='float64', ndmin=2) if bids.shape[1] != 4: bids = bids.reshape((0, 4)) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py index 762db1c8e5..3de3716210 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py @@ -75,9 +75,10 @@ async def get_order_book_data(trading_pair: str) -> Dict[str, any]: params={"limit": 150, "symbols": ex_pair}) return orderbook_response[ex_pair] except HitbtcAPIError as e: + err = e.error_payload.get('error', e.error_payload) raise IOError( f"Error fetching OrderBook for {trading_pair} at {Constants.EXCHANGE_NAME}. " - f"HTTP status is {e.error_payload['status']}. Error is {e.error_payload['error']}.") + f"HTTP status is {e.error_payload['status']}. Error is {err.get('message', str(err))}.") async def get_new_order_book(self, trading_pair: str) -> OrderBook: snapshot: Dict[str, Any] = await self.get_order_book_data(trading_pair) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index 70f96fbe3b..a677d70728 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -460,7 +460,7 @@ async def _create_order(self, except asyncio.CancelledError: raise except HitbtcAPIError as e: - error_reason = str(e.error_payload['error']) + error_reason = e.error_payload.get('error', {}).get('message') self.stop_tracking_order(order_id) self.logger().network( f"Error submitting {trade_type.name} {order_type.name} order to {Constants.EXCHANGE_NAME} for " @@ -524,10 +524,9 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: except asyncio.CancelledError: raise except HitbtcAPIError as e: - err = e.error_payload['error'] - error_reason = err['message'] if 'message' in err else err + err = e.error_payload.get('error', e.error_payload) self._order_not_found_records[order_id] = self._order_not_found_records.get(order_id, 0) + 1 - if err['code'] == 20002 and \ + if err.get('code') == 20002 and \ self._order_not_found_records[order_id] >= self.ORDER_NOT_EXIST_CANCEL_COUNT: order_was_cancelled = True if order_was_cancelled: @@ -538,7 +537,7 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: return CancellationResult(order_id, True) else: self.logger().network( - f"Failed to cancel order {order_id}: {error_reason}", + f"Failed to cancel order {order_id}: {err.get('message', str(err))}", exc_info=True, app_warning_msg=f"Failed to cancel the order {order_id} on {Constants.EXCHANGE_NAME}. " f"Check API key and network connection." @@ -608,7 +607,8 @@ async def _update_order_status(self): for response, tracked_order in zip(responses, tracked_orders): client_order_id = tracked_order.client_order_id if isinstance(response, HitbtcAPIError): - if response.error_payload['error']['code'] == 20002: + err = response.error_payload.get('error', response.error_payload) + if err.get('code') == 20002: self._order_not_found_records[client_order_id] = \ self._order_not_found_records.get(client_order_id, 0) + 1 if self._order_not_found_records[client_order_id] < self.ORDER_NOT_EXIST_CONFIRMATION_COUNT: diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py index 8658c56acf..1f0bc1d631 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py @@ -11,6 +11,9 @@ OrderBookMessage, OrderBookMessageType, ) +from .hitbtc_utils import ( + convert_from_exchange_trading_pair, +) class HitbtcOrderBookMessage(OrderBookMessage): @@ -46,10 +49,10 @@ def trade_id(self) -> int: @property def trading_pair(self) -> str: - if "symbol" in self.content: - return self.content["symbol"] - elif "trading_pair" in self.content: + if "trading_pair" in self.content: return self.content["trading_pair"] + elif "symbol" in self.content: + return convert_from_exchange_trading_pair(self.content["symbol"]) @property def asks(self) -> List[OrderBookRow]: diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py index 55bdce0d43..d3161de17e 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_tracker.py @@ -83,8 +83,7 @@ async def _track_single_book(self, trading_pair: str): # Output some statistics periodically. now: float = time.time() if int(now / 60.0) > int(last_message_timestamp / 60.0): - self.logger().debug("Processed %d order book diffs for %s.", - diff_messages_accepted, trading_pair) + self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") diff_messages_accepted = 0 last_message_timestamp = now elif message.type is OrderBookMessageType.SNAPSHOT: @@ -98,7 +97,7 @@ async def _track_single_book(self, trading_pair: str): d_bids, d_asks = active_order_tracker.convert_diff_message_to_order_book_row(diff_message) order_book.apply_diffs(d_bids, d_asks, diff_message.update_id) - self.logger().debug("Processed order book snapshot for %s.", trading_pair) + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") except asyncio.CancelledError: raise except Exception: From 448b70d0c9fb51837508d493bd5d5d720f050a63 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Tue, 9 Mar 2021 13:07:22 +0000 Subject: [PATCH 20/33] HitBTC: Fix default fees --- hummingbot/connector/exchange/hitbtc/hitbtc_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py index d48125020b..ad17938251 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py @@ -23,7 +23,7 @@ EXAMPLE_PAIR = "ETH-USDT" -DEFAULT_FEES = [0.1, 0.1] +DEFAULT_FEES = [0.1, 0.25] class HitbtcAPIError(IOError): From 6205d7d013d3c96b8f034096557e5e3568c8b6d5 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Tue, 9 Mar 2021 19:27:09 +0000 Subject: [PATCH 21/33] HitBTC: Add Tests and couple of fixes --- .../exchange/hitbtc/hitbtc_exchange.py | 4 +- test/connector/exchange/hitbtc/.gitignore | 1 + test/connector/exchange/hitbtc/__init__.py | 0 .../exchange/hitbtc/test_hitbtc_auth.py | 55 +++ .../exchange/hitbtc/test_hitbtc_exchange.py | 438 ++++++++++++++++++ .../hitbtc/test_hitbtc_order_book_tracker.py | 103 ++++ .../hitbtc/test_hitbtc_user_stream_tracker.py | 37 ++ 7 files changed, 637 insertions(+), 1 deletion(-) create mode 100644 test/connector/exchange/hitbtc/.gitignore create mode 100644 test/connector/exchange/hitbtc/__init__.py create mode 100644 test/connector/exchange/hitbtc/test_hitbtc_auth.py create mode 100644 test/connector/exchange/hitbtc/test_hitbtc_exchange.py create mode 100755 test/connector/exchange/hitbtc/test_hitbtc_order_book_tracker.py create mode 100644 test/connector/exchange/hitbtc/test_hitbtc_user_stream_tracker.py diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index a677d70728..6fafb39ce2 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -531,9 +531,10 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: order_was_cancelled = True if order_was_cancelled: self.logger().info(f"Successfully cancelled order {order_id} on {Constants.EXCHANGE_NAME}.") - self.stop_tracking_order(order_id) self.trigger_event(MarketEvent.OrderCancelled, OrderCancelledEvent(self.current_timestamp, order_id)) + tracked_order.cancelled_event.set() + self.stop_tracking_order(order_id) return CancellationResult(order_id, True) else: self.logger().network( @@ -732,6 +733,7 @@ async def _process_trade_message(self, trade_msg: Dict[str, Any]): else MarketEvent.SellOrderCompleted event_class = BuyOrderCompletedEvent if tracked_order.trade_type is TradeType.BUY \ else SellOrderCompletedEvent + await asyncio.sleep(0.1) self.trigger_event(event_tag, event_class(self.current_timestamp, tracked_order.client_order_id, diff --git a/test/connector/exchange/hitbtc/.gitignore b/test/connector/exchange/hitbtc/.gitignore new file mode 100644 index 0000000000..23d9952b8c --- /dev/null +++ b/test/connector/exchange/hitbtc/.gitignore @@ -0,0 +1 @@ +backups \ No newline at end of file diff --git a/test/connector/exchange/hitbtc/__init__.py b/test/connector/exchange/hitbtc/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/connector/exchange/hitbtc/test_hitbtc_auth.py b/test/connector/exchange/hitbtc/test_hitbtc_auth.py new file mode 100644 index 0000000000..6cc71c27e9 --- /dev/null +++ b/test/connector/exchange/hitbtc/test_hitbtc_auth.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +import sys +import asyncio +import unittest +import aiohttp +import conf +import logging +from os.path import join, realpath +from typing import Dict, Any +from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth +from hummingbot.connector.exchange.hitbtc.hitbtc_websocket import HitbtcWebsocket +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants + +sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +logging.basicConfig(level=METRICS_LOG_LEVEL) + + +class TestAuth(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + api_key = conf.hitbtc_api_key + secret_key = conf.hitbtc_secret_key + cls.auth = HitbtcAuth(api_key, secret_key) + + async def rest_auth(self) -> Dict[Any, Any]: + endpoint = Constants.ENDPOINT['USER_BALANCES'] + headers = self.auth.get_headers("GET", f"{Constants.REST_URL_AUTH}/{endpoint}", None) + response = await aiohttp.ClientSession().get(f"{Constants.REST_URL}/{endpoint}", headers=headers) + return await response.json() + + async def ws_auth(self) -> Dict[Any, Any]: + ws = HitbtcWebsocket(self.auth) + await ws.connect() + await ws.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"], None, {}) + async for response in ws.on_message(): + return response + + def test_rest_auth(self): + result = self.ev_loop.run_until_complete(self.rest_auth()) + if len(result) == 0 or "currency" not in result[0].keys(): + print(f"Unexpected response for API call: {result}") + assert "currency" in result[0].keys() + + def test_ws_auth(self): + try: + response = self.ev_loop.run_until_complete(self.ws_auth()) + no_errors = True + except Exception: + no_errors = False + assert no_errors is True + if 'result' not in response: + print(f"Unexpected response for API call: {response}") + assert response['result'] is True diff --git a/test/connector/exchange/hitbtc/test_hitbtc_exchange.py b/test/connector/exchange/hitbtc/test_hitbtc_exchange.py new file mode 100644 index 0000000000..0456f5a8a9 --- /dev/null +++ b/test/connector/exchange/hitbtc/test_hitbtc_exchange.py @@ -0,0 +1,438 @@ +from os.path import join, realpath +import sys; sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +import asyncio +import logging +from decimal import Decimal +import unittest +import contextlib +import time +import os +from typing import List +import conf +import math + +from hummingbot.core.clock import Clock, ClockMode +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL +from hummingbot.core.utils.async_utils import safe_gather, safe_ensure_future +from hummingbot.core.event.event_logger import EventLogger +from hummingbot.core.event.events import ( + BuyOrderCompletedEvent, + BuyOrderCreatedEvent, + MarketEvent, + OrderFilledEvent, + OrderType, + SellOrderCompletedEvent, + SellOrderCreatedEvent, + OrderCancelledEvent +) +from hummingbot.model.sql_connection_manager import ( + SQLConnectionManager, + SQLConnectionType +) +from hummingbot.model.market_state import MarketState +from hummingbot.model.order import Order +from hummingbot.model.trade_fill import TradeFill +from hummingbot.connector.markets_recorder import MarketsRecorder +from hummingbot.connector.exchange.hitbtc.hitbtc_exchange import HitbtcExchange + +logging.basicConfig(level=METRICS_LOG_LEVEL) + +API_KEY = conf.hitbtc_api_key +API_SECRET = conf.hitbtc_secret_key + + +class HitbtcExchangeUnitTest(unittest.TestCase): + events: List[MarketEvent] = [ + MarketEvent.BuyOrderCompleted, + MarketEvent.SellOrderCompleted, + MarketEvent.OrderFilled, + MarketEvent.TransactionFailure, + MarketEvent.BuyOrderCreated, + MarketEvent.SellOrderCreated, + MarketEvent.OrderCancelled, + MarketEvent.OrderFailure + ] + connector: HitbtcExchange + event_logger: EventLogger + trading_pair = "BTC-USD" + base_token, quote_token = trading_pair.split("-") + stack: contextlib.ExitStack + + @classmethod + def setUpClass(cls): + global MAINNET_RPC_URL + + cls.ev_loop = asyncio.get_event_loop() + + cls.clock: Clock = Clock(ClockMode.REALTIME) + cls.connector: HitbtcExchange = HitbtcExchange( + hitbtc_api_key=API_KEY, + hitbtc_secret_key=API_SECRET, + trading_pairs=[cls.trading_pair], + trading_required=True + ) + print("Initializing Hitbtc market... this will take about a minute.") + cls.clock.add_iterator(cls.connector) + cls.stack: contextlib.ExitStack = contextlib.ExitStack() + cls._clock = cls.stack.enter_context(cls.clock) + cls.ev_loop.run_until_complete(cls.wait_til_ready()) + print("Ready.") + + @classmethod + def tearDownClass(cls) -> None: + cls.stack.close() + + @classmethod + async def wait_til_ready(cls, connector = None): + if connector is None: + connector = cls.connector + while True: + now = time.time() + next_iteration = now // 1.0 + 1 + if connector.ready: + break + else: + await cls._clock.run_til(next_iteration) + await asyncio.sleep(1.0) + + def setUp(self): + self.db_path: str = realpath(join(__file__, "../connector_test.sqlite")) + try: + os.unlink(self.db_path) + except FileNotFoundError: + pass + + self.event_logger = EventLogger() + for event_tag in self.events: + self.connector.add_listener(event_tag, self.event_logger) + + def tearDown(self): + for event_tag in self.events: + self.connector.remove_listener(event_tag, self.event_logger) + self.event_logger = None + + async def run_parallel_async(self, *tasks): + future: asyncio.Future = safe_ensure_future(safe_gather(*tasks)) + while not future.done(): + now = time.time() + next_iteration = now // 1.0 + 1 + await self._clock.run_til(next_iteration) + await asyncio.sleep(1.0) + return future.result() + + def run_parallel(self, *tasks): + return self.ev_loop.run_until_complete(self.run_parallel_async(*tasks)) + + def _place_order(self, is_buy, amount, order_type, price, ex_order_id) -> str: + if is_buy: + cl_order_id = self.connector.buy(self.trading_pair, amount, order_type, price) + else: + cl_order_id = self.connector.sell(self.trading_pair, amount, order_type, price) + return cl_order_id + + def _cancel_order(self, cl_order_id, connector=None): + if connector is None: + connector = self.connector + return connector.cancel(self.trading_pair, cl_order_id) + + def test_estimate_fee(self): + maker_fee = self.connector.estimate_fee_pct(True) + self.assertAlmostEqual(maker_fee, Decimal("0.001")) + taker_fee = self.connector.estimate_fee_pct(False) + self.assertAlmostEqual(taker_fee, Decimal("0.0025")) + + def test_buy_and_sell(self): + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.02") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + quote_bal = self.connector.get_available_balance(self.quote_token) + base_bal = self.connector.get_available_balance(self.base_token) + + order_id = self._place_order(True, amount, OrderType.LIMIT, price, 1) + order_completed_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(5)) + trade_events = [t for t in self.event_logger.event_log if isinstance(t, OrderFilledEvent)] + base_amount_traded = sum(t.amount for t in trade_events) + quote_amount_traded = sum(t.amount * t.price for t in trade_events) + + self.assertTrue([evt.order_type == OrderType.LIMIT for evt in trade_events]) + self.assertEqual(order_id, order_completed_event.order_id) + self.assertEqual(amount, order_completed_event.base_asset_amount) + self.assertEqual("BTC", order_completed_event.base_asset) + self.assertEqual("USD", order_completed_event.quote_asset) + self.assertAlmostEqual(base_amount_traded, order_completed_event.base_asset_amount) + self.assertAlmostEqual(quote_amount_traded, order_completed_event.quote_asset_amount) + self.assertGreater(order_completed_event.fee_amount, Decimal(0)) + self.assertTrue(any([isinstance(event, BuyOrderCreatedEvent) and str(event.order_id) == str(order_id) + for event in self.event_logger.event_log])) + + # check available quote balance gets updated, we need to wait a bit for the balance message to arrive + expected_quote_bal = quote_bal - quote_amount_traded + # self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.ev_loop.run_until_complete(self.connector._update_balances()) + self.assertAlmostEqual(expected_quote_bal, self.connector.get_available_balance(self.quote_token), 1) + + # Reset the logs + self.event_logger.clear() + + # Try to sell back the same amount to the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.98") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + order_id = self._place_order(False, amount, OrderType.LIMIT, price, 2) + order_completed_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCompletedEvent)) + trade_events = [t for t in self.event_logger.event_log if isinstance(t, OrderFilledEvent)] + base_amount_traded = sum(t.amount for t in trade_events) + quote_amount_traded = sum(t.amount * t.price for t in trade_events) + + self.assertTrue([evt.order_type == OrderType.LIMIT for evt in trade_events]) + self.assertEqual(order_id, order_completed_event.order_id) + self.assertEqual(amount, order_completed_event.base_asset_amount) + self.assertEqual("BTC", order_completed_event.base_asset) + self.assertEqual("USD", order_completed_event.quote_asset) + self.assertAlmostEqual(base_amount_traded, order_completed_event.base_asset_amount) + self.assertAlmostEqual(quote_amount_traded, order_completed_event.quote_asset_amount) + self.assertGreater(order_completed_event.fee_amount, Decimal(0)) + self.assertTrue(any([isinstance(event, SellOrderCreatedEvent) and event.order_id == order_id + for event in self.event_logger.event_log])) + + # check available base balance gets updated, we need to wait a bit for the balance message to arrive + expected_base_bal = base_bal + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.ev_loop.run_until_complete(self.connector._update_balances()) + self.ev_loop.run_until_complete(asyncio.sleep(5)) + self.assertAlmostEqual(expected_base_bal, self.connector.get_available_balance(self.base_token), 5) + + def test_limit_makers_unfilled(self): + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.8") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.ev_loop.run_until_complete(self.connector._update_balances()) + self.ev_loop.run_until_complete(asyncio.sleep(2)) + quote_bal = self.connector.get_available_balance(self.quote_token) + + cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + # check available quote balance gets updated, we need to wait a bit for the balance message to arrive + taker_fee = self.connector.estimate_fee_pct(False) + quote_amount = ((price * amount)) + quote_amount = ((price * amount) * (Decimal("1") + taker_fee)) + expected_quote_bal = quote_bal - quote_amount + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.ev_loop.run_until_complete(self.connector._update_balances()) + self.ev_loop.run_until_complete(asyncio.sleep(2)) + + self.assertAlmostEqual(expected_quote_bal, self.connector.get_available_balance(self.quote_token), 5) + self._cancel_order(cl_order_id) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.2") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + cl_order_id = self._place_order(False, amount, OrderType.LIMIT_MAKER, price, 2) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + self._cancel_order(cl_order_id) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + # # @TODO: find a way to create "rejected" + # def test_limit_maker_rejections(self): + # price = self.connector.get_price(self.trading_pair, True) * Decimal("1.2") + # price = self.connector.quantize_order_price(self.trading_pair, price) + # amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.000001")) + # cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1) + # event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + # self.assertEqual(cl_order_id, event.order_id) + + # price = self.connector.get_price(self.trading_pair, False) * Decimal("0.8") + # price = self.connector.quantize_order_price(self.trading_pair, price) + # amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.000001")) + # cl_order_id = self._place_order(False, amount, OrderType.LIMIT_MAKER, price, 2) + # event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + # self.assertEqual(cl_order_id, event.order_id) + + def test_cancel_all(self): + bid_price = self.connector.get_price(self.trading_pair, True) + ask_price = self.connector.get_price(self.trading_pair, False) + bid_price = self.connector.quantize_order_price(self.trading_pair, bid_price * Decimal("0.9")) + ask_price = self.connector.quantize_order_price(self.trading_pair, ask_price * Decimal("1.1")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + buy_id = self._place_order(True, amount, OrderType.LIMIT, bid_price, 1) + sell_id = self._place_order(False, amount, OrderType.LIMIT, ask_price, 2) + + self.ev_loop.run_until_complete(asyncio.sleep(1)) + asyncio.ensure_future(self.connector.cancel_all(5)) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + cancel_events = [t for t in self.event_logger.event_log if isinstance(t, OrderCancelledEvent)] + self.assertEqual({buy_id, sell_id}, {o.order_id for o in cancel_events}) + + def test_order_quantized_values(self): + bid_price: Decimal = self.connector.get_price(self.trading_pair, True) + ask_price: Decimal = self.connector.get_price(self.trading_pair, False) + mid_price: Decimal = (bid_price + ask_price) / 2 + + # Make sure there's enough balance to make the limit orders. + self.assertGreater(self.connector.get_balance("BTC"), Decimal("0.0005")) + self.assertGreater(self.connector.get_balance("USD"), Decimal("10")) + + # Intentionally set some prices with too many decimal places s.t. they + # need to be quantized. Also, place them far away from the mid-price s.t. they won't + # get filled during the test. + bid_price = self.connector.quantize_order_price(self.trading_pair, mid_price * Decimal("0.9333192292111341")) + ask_price = self.connector.quantize_order_price(self.trading_pair, mid_price * Decimal("1.1492431474884933")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.000123456")) + + # Test bid order + cl_order_id_1 = self._place_order(True, amount, OrderType.LIMIT, bid_price, 1) + # Wait for the order created event and examine the order made + self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + + # Test ask order + cl_order_id_2 = self._place_order(False, amount, OrderType.LIMIT, ask_price, 1) + # Wait for the order created event and examine and order made + self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCreatedEvent)) + + self._cancel_order(cl_order_id_1) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self._cancel_order(cl_order_id_2) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + + def test_orders_saving_and_restoration(self): + config_path = "test_config" + strategy_name = "test_strategy" + sql = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, db_path=self.db_path) + order_id = None + recorder = MarketsRecorder(sql, [self.connector], config_path, strategy_name) + recorder.start() + + try: + self.connector._in_flight_orders.clear() + self.assertEqual(0, len(self.connector.tracking_states)) + + # Try to put limit buy order for 0.02 ETH worth of ZRX, and watch for order creation event. + current_bid_price: Decimal = self.connector.get_price(self.trading_pair, True) + price: Decimal = current_bid_price * Decimal("0.8") + price = self.connector.quantize_order_price(self.trading_pair, price) + + amount: Decimal = Decimal("0.0001") + amount = self.connector.quantize_order_amount(self.trading_pair, amount) + + cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + + # Verify tracking states + self.assertEqual(1, len(self.connector.tracking_states)) + self.assertEqual(cl_order_id, list(self.connector.tracking_states.keys())[0]) + + # Verify orders from recorder + recorded_orders: List[Order] = recorder.get_orders_for_config_and_market(config_path, self.connector) + self.assertEqual(1, len(recorded_orders)) + self.assertEqual(cl_order_id, recorded_orders[0].id) + + # Verify saved market states + saved_market_states: MarketState = recorder.get_market_states(config_path, self.connector) + self.assertIsNotNone(saved_market_states) + self.assertIsInstance(saved_market_states.saved_state, dict) + self.assertGreater(len(saved_market_states.saved_state), 0) + + # Close out the current market and start another market. + self.connector.stop(self._clock) + self.ev_loop.run_until_complete(asyncio.sleep(5)) + self.clock.remove_iterator(self.connector) + for event_tag in self.events: + self.connector.remove_listener(event_tag, self.event_logger) + # Clear the event loop + self.event_logger.clear() + new_connector = HitbtcExchange(API_KEY, API_SECRET, [self.trading_pair], True) + for event_tag in self.events: + new_connector.add_listener(event_tag, self.event_logger) + recorder.stop() + recorder = MarketsRecorder(sql, [new_connector], config_path, strategy_name) + recorder.start() + saved_market_states = recorder.get_market_states(config_path, new_connector) + self.clock.add_iterator(new_connector) + self.ev_loop.run_until_complete(self.wait_til_ready(new_connector)) + self.assertEqual(0, len(new_connector.limit_orders)) + self.assertEqual(0, len(new_connector.tracking_states)) + new_connector.restore_tracking_states(saved_market_states.saved_state) + self.assertEqual(1, len(new_connector.limit_orders)) + self.assertEqual(1, len(new_connector.tracking_states)) + + # Cancel the order and verify that the change is saved. + self._cancel_order(cl_order_id, new_connector) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + recorder.save_market_states(config_path, new_connector) + order_id = None + self.assertEqual(0, len(new_connector.limit_orders)) + self.assertEqual(0, len(new_connector.tracking_states)) + saved_market_states = recorder.get_market_states(config_path, new_connector) + self.assertEqual(0, len(saved_market_states.saved_state)) + finally: + if order_id is not None: + self.connector.cancel(self.trading_pair, cl_order_id) + self.run_parallel(self.event_logger.wait_for(OrderCancelledEvent)) + + recorder.stop() + os.unlink(self.db_path) + + def test_update_last_prices(self): + # This is basic test to see if order_book last_trade_price is initiated and updated. + for order_book in self.connector.order_books.values(): + for _ in range(5): + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.assertFalse(math.isnan(order_book.last_trade_price)) + + def test_filled_orders_recorded(self): + config_path: str = "test_config" + strategy_name: str = "test_strategy" + sql = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, db_path=self.db_path) + order_id = None + recorder = MarketsRecorder(sql, [self.connector], config_path, strategy_name) + recorder.start() + + try: + # Try to buy some token from the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.05") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + order_id = self._place_order(True, amount, OrderType.LIMIT, price, 1) + self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + + # Reset the logs + self.event_logger.clear() + + # Try to sell back the same amount to the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.95") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + order_id = self._place_order(False, amount, OrderType.LIMIT, price, 2) + self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + + # Query the persisted trade logs + trade_fills: List[TradeFill] = recorder.get_trades_for_config(config_path) + self.assertGreaterEqual(len(trade_fills), 2) + buy_fills: List[TradeFill] = [t for t in trade_fills if t.trade_type == "BUY"] + sell_fills: List[TradeFill] = [t for t in trade_fills if t.trade_type == "SELL"] + self.assertGreaterEqual(len(buy_fills), 1) + self.assertGreaterEqual(len(sell_fills), 1) + + order_id = None + + finally: + if order_id is not None: + self.connector.cancel(self.trading_pair, order_id) + self.run_parallel(self.event_logger.wait_for(OrderCancelledEvent)) + + recorder.stop() + os.unlink(self.db_path) diff --git a/test/connector/exchange/hitbtc/test_hitbtc_order_book_tracker.py b/test/connector/exchange/hitbtc/test_hitbtc_order_book_tracker.py new file mode 100755 index 0000000000..ae3778e7c9 --- /dev/null +++ b/test/connector/exchange/hitbtc/test_hitbtc_order_book_tracker.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +import sys +import math +import time +import asyncio +import logging +import unittest +from os.path import join, realpath +from typing import Dict, Optional, List +from hummingbot.core.event.event_logger import EventLogger +from hummingbot.core.event.events import OrderBookEvent, OrderBookTradeEvent, TradeType +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_tracker import HitbtcOrderBookTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_api_order_book_data_source import HitbtcAPIOrderBookDataSource +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL + + +sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +logging.basicConfig(level=METRICS_LOG_LEVEL) + + +class HitbtcOrderBookTrackerUnitTest(unittest.TestCase): + order_book_tracker: Optional[HitbtcOrderBookTracker] = None + events: List[OrderBookEvent] = [ + OrderBookEvent.TradeEvent + ] + trading_pairs: List[str] = [ + "BTC-USD", + "ETH-USD", + ] + + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + cls.order_book_tracker: HitbtcOrderBookTracker = HitbtcOrderBookTracker(cls.trading_pairs) + cls.order_book_tracker.start() + cls.ev_loop.run_until_complete(cls.wait_til_tracker_ready()) + + @classmethod + async def wait_til_tracker_ready(cls): + while True: + if len(cls.order_book_tracker.order_books) > 0: + print("Initialized real-time order books.") + return + await asyncio.sleep(1) + + async def run_parallel_async(self, *tasks, timeout=None): + future: asyncio.Future = asyncio.ensure_future(asyncio.gather(*tasks)) + timer = 0 + while not future.done(): + if timeout and timer > timeout: + raise Exception("Timeout running parallel async tasks in tests") + timer += 1 + now = time.time() + _next_iteration = now // 1.0 + 1 # noqa: F841 + await asyncio.sleep(1.0) + return future.result() + + def run_parallel(self, *tasks): + return self.ev_loop.run_until_complete(self.run_parallel_async(*tasks)) + + def setUp(self): + self.event_logger = EventLogger() + for event_tag in self.events: + for trading_pair, order_book in self.order_book_tracker.order_books.items(): + order_book.add_listener(event_tag, self.event_logger) + + def test_order_book_trade_event_emission(self): + """ + Tests if the order book tracker is able to retrieve order book trade message from exchange and emit order book + trade events after correctly parsing the trade messages + """ + self.run_parallel(self.event_logger.wait_for(OrderBookTradeEvent)) + for ob_trade_event in self.event_logger.event_log: + self.assertTrue(type(ob_trade_event) == OrderBookTradeEvent) + self.assertTrue(ob_trade_event.trading_pair in self.trading_pairs) + self.assertTrue(type(ob_trade_event.timestamp) in [float, int]) + self.assertTrue(type(ob_trade_event.amount) == float) + self.assertTrue(type(ob_trade_event.price) == float) + self.assertTrue(type(ob_trade_event.type) == TradeType) + # datetime is in seconds + self.assertTrue(math.ceil(math.log10(ob_trade_event.timestamp)) == 10) + self.assertTrue(ob_trade_event.amount > 0) + self.assertTrue(ob_trade_event.price > 0) + + def test_tracker_integrity(self): + # Wait 5 seconds to process some diffs. + self.ev_loop.run_until_complete(asyncio.sleep(5.0)) + order_books: Dict[str, OrderBook] = self.order_book_tracker.order_books + eth_usd: OrderBook = order_books["ETH-USD"] + self.assertIsNot(eth_usd.last_diff_uid, 0) + self.assertGreaterEqual(eth_usd.get_price_for_volume(True, 10).result_price, + eth_usd.get_price(True)) + self.assertLessEqual(eth_usd.get_price_for_volume(False, 10).result_price, + eth_usd.get_price(False)) + + def test_api_get_last_traded_prices(self): + prices = self.ev_loop.run_until_complete( + HitbtcAPIOrderBookDataSource.get_last_traded_prices(["BTC-USD", "LTC-BTC"])) + for key, value in prices.items(): + print(f"{key} last_trade_price: {value}") + self.assertGreater(prices["BTC-USD"], 1000) + self.assertLess(prices["LTC-BTC"], 1) diff --git a/test/connector/exchange/hitbtc/test_hitbtc_user_stream_tracker.py b/test/connector/exchange/hitbtc/test_hitbtc_user_stream_tracker.py new file mode 100644 index 0000000000..5c82f2372b --- /dev/null +++ b/test/connector/exchange/hitbtc/test_hitbtc_user_stream_tracker.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +import sys +import asyncio +import logging +import unittest +import conf + +from os.path import join, realpath +from hummingbot.connector.exchange.hitbtc.hitbtc_user_stream_tracker import HitbtcUserStreamTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth +from hummingbot.core.utils.async_utils import safe_ensure_future +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL + + +sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +logging.basicConfig(level=METRICS_LOG_LEVEL) + + +class HitbtcUserStreamTrackerUnitTest(unittest.TestCase): + api_key = conf.hitbtc_api_key + api_secret = conf.hitbtc_secret_key + + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + cls.trading_pairs = ["BTC-USD"] + cls.user_stream_tracker: HitbtcUserStreamTracker = HitbtcUserStreamTracker( + hitbtc_auth=HitbtcAuth(cls.api_key, cls.api_secret), + trading_pairs=cls.trading_pairs) + cls.user_stream_tracker_task: asyncio.Task = safe_ensure_future(cls.user_stream_tracker.start()) + + def test_user_stream(self): + # Wait process some msgs. + print("Sleeping for 30s to gather some user stream messages.") + self.ev_loop.run_until_complete(asyncio.sleep(30.0)) + print(self.user_stream_tracker.user_stream) From 2d53ead60ae35bae0a8b092e388be46513e8d40c Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Thu, 11 Mar 2021 02:07:28 +0000 Subject: [PATCH 22/33] HitBTC: Fix stuck bids on order book --- .../connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx index 6d4b57b836..5ec8362df3 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx @@ -116,7 +116,7 @@ cdef class HitbtcActiveOrderTracker: float(price), float(self._active_bids[price]), message.update_id] - for price in sorted(self._active_bids.keys(), reverse=True)], dtype='float64', ndmin=2) + for price in sorted(self._active_bids.keys())], dtype='float64', ndmin=2) np.ndarray[np.float64_t, ndim=2] asks = np.array( [[message.timestamp, float(price), From 9aacb176d0e9539d254f0d83e186dc491cad66ce Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Thu, 11 Mar 2021 11:12:52 +0000 Subject: [PATCH 23/33] HitBTC: Change order of cancel events --- hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index 6fafb39ce2..7dd3075930 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -531,10 +531,10 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: order_was_cancelled = True if order_was_cancelled: self.logger().info(f"Successfully cancelled order {order_id} on {Constants.EXCHANGE_NAME}.") + self.stop_tracking_order(order_id) self.trigger_event(MarketEvent.OrderCancelled, OrderCancelledEvent(self.current_timestamp, order_id)) tracked_order.cancelled_event.set() - self.stop_tracking_order(order_id) return CancellationResult(order_id, True) else: self.logger().network( @@ -662,10 +662,10 @@ def _process_order_message(self, order_msg: Dict[str, Any]): if tracked_order.is_cancelled: self.logger().info(f"Successfully cancelled order {client_order_id}.") + self.stop_tracking_order(client_order_id) self.trigger_event(MarketEvent.OrderCancelled, OrderCancelledEvent(self.current_timestamp, client_order_id)) tracked_order.cancelled_event.set() - self.stop_tracking_order(client_order_id) elif tracked_order.is_failure: self.logger().info(f"The market order {client_order_id} has failed according to order status API. ") self.trigger_event(MarketEvent.OrderFailure, From 6f20d0e92b56d4d327d93e3261671719f7dc7bd8 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Thu, 11 Mar 2021 18:15:27 +0000 Subject: [PATCH 24/33] HitBTC: Fix intermittent user stream listener loop Fixes: `Unexpected error in user stream listener loop.` `RuntimeError: dictionary changed size during iteration` --- hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index 7dd3075930..94fc62e247 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -699,9 +699,10 @@ async def _process_trade_message(self, trade_msg: Dict[str, Any]): "tradeFee": "-0.000000005" } """ - for order in self._in_flight_orders.values(): + tracked_orders = list(self._in_flight_orders.values()) + for order in tracked_orders: await order.get_exchange_order_id() - track_order = [o for o in self._in_flight_orders.values() if trade_msg["id"] == o.exchange_order_id] + track_order = [o for o in tracked_orders if trade_msg["id"] == o.exchange_order_id] if not track_order: return tracked_order = track_order[0] From b2fb4eff4228f354098710f51079179439cafb32 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Thu, 18 Mar 2021 23:14:13 +0000 Subject: [PATCH 25/33] HitBTC: Housekeeping --- .../hitbtc/hitbtc_active_order_tracker.pyx | 9 ++++--- .../hitbtc_api_order_book_data_source.py | 3 +-- .../hitbtc_api_user_stream_data_source.py | 5 ++-- .../exchange/hitbtc/hitbtc_in_flight_order.py | 9 ------- .../connector/exchange/hitbtc/hitbtc_utils.py | 25 +------------------ 5 files changed, 9 insertions(+), 42 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx index 5ec8362df3..ea918d239c 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx @@ -1,9 +1,7 @@ # distutils: language=c++ # distutils: sources=hummingbot/core/cpp/OrderBookEntry.cpp - import logging import numpy as np - from decimal import Decimal from typing import Dict from hummingbot.logger import HummingbotLogger @@ -51,6 +49,7 @@ cdef class HitbtcActiveOrderTracker: cdef tuple c_convert_diff_message_to_np_arrays(self, object message): cdef: dict content = message.content + list content_keys = list(content.keys()) list bid_entries = [] list ask_entries = [] str order_id @@ -61,8 +60,10 @@ cdef class HitbtcActiveOrderTracker: double timestamp = message.timestamp double amount = 0 - bid_entries = content["bid"] - ask_entries = content["ask"] + if "bids" in content_keys: + bid_entries = content["bid"] + if "asks" in content_keys: + ask_entries = content["ask"] bids = s_empty_diff asks = s_empty_diff diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py index 3de3716210..40d83516da 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_order_book_data_source.py @@ -8,7 +8,6 @@ from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.order_book_message import OrderBookMessage from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource -# from hummingbot.core.utils.async_utils import safe_gather from hummingbot.logger import HummingbotLogger from .hitbtc_constants import Constants from .hitbtc_active_order_tracker import HitbtcActiveOrderTracker @@ -60,7 +59,7 @@ async def fetch_trading_pairs() -> List[str]: # Filter out unmatched pairs so nothing breaks return [sym for sym in trading_pairs if sym is not None] except Exception: - # Do nothing if the request fails -- there will be no autocomplete for huobi trading pairs + # Do nothing if the request fails -- there will be no autocomplete for HitBTC trading pairs pass return [] diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py index af67aed70b..84b051dbfc 100755 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py @@ -1,5 +1,4 @@ #!/usr/bin/env python - import time import asyncio import logging @@ -51,11 +50,11 @@ async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: await ws.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"], None, {}) async for msg in ws.on_message(): - yield msg self._last_recv_time = time.time() - if (msg.get("params") is None): + if msg.get("params") is None: continue + yield msg except Exception as e: raise e finally: diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py b/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py index 22ce9e18fc..54766be2f1 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_in_flight_order.py @@ -49,15 +49,6 @@ def is_failure(self) -> bool: def is_cancelled(self) -> bool: return self.last_state in {"canceled", "expired"} - # @property - # def order_type_description(self) -> str: - # """ - # :return: Order description string . One of ["limit buy" / "limit sell" / "market buy" / "market sell"] - # """ - # order_type = "market" if self.order_type is OrderType.MARKET else "limit" - # side = "buy" if self.trade_type == TradeType.BUY else "sell" - # return f"{order_type} {side}" - @classmethod def from_json(cls, data: Dict[str, Any]) -> InFlightOrderBase: """ diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py index ad17938251..c549ce8b72 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py @@ -6,7 +6,6 @@ from typing import ( Any, Dict, - List, Optional, Tuple, ) @@ -21,7 +20,7 @@ CENTRALIZED = True -EXAMPLE_PAIR = "ETH-USDT" +EXAMPLE_PAIR = "BTC-USD" DEFAULT_FEES = [0.1, 0.25] @@ -32,24 +31,6 @@ def __init__(self, error_payload: Dict[str, Any]): self.error_payload = error_payload -# deeply merge two dictionaries -def merge_dicts(source: Dict, destination: Dict) -> Dict: - for key, value in source.items(): - if isinstance(value, dict): - # get node or create one - node = destination.setdefault(key, {}) - merge_dicts(value, node) - else: - destination[key] = value - - return destination - - -# join paths -def join_paths(*paths: List[str]) -> str: - return "/".join(paths) - - # convert date string to timestamp def str_date_to_ts(date: str) -> int: return int(dateparse(date).timestamp()) @@ -100,10 +81,6 @@ def get_new_client_order_id(is_buy: bool, trading_pair: str) -> str: return f"{Constants.HBOT_BROKER_ID}-{side}-{base_str}{quote_str}-{get_tracking_nonce()}" -def get_api_reason(code: str) -> str: - return Constants.API_REASONS.get(int(code), code) - - def retry_sleep_time(try_count: int) -> float: random.seed() randSleep = 1 + float(random.randint(1, 10) / 100) From 70ad0a2bad26faf10942f34d792f1a0125834248 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Fri, 19 Mar 2021 03:44:12 +0000 Subject: [PATCH 26/33] HitBTC: WS Clean up and report auth errors --- .../hitbtc_api_user_stream_data_source.py | 15 +++++++---- .../exchange/hitbtc/hitbtc_exchange.py | 27 ++++++++++--------- .../exchange/hitbtc/hitbtc_websocket.py | 11 +++++--- 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py index 84b051dbfc..77d09fa9a8 100755 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py @@ -12,6 +12,7 @@ from hummingbot.logger import HummingbotLogger from .hitbtc_constants import Constants from .hitbtc_auth import HitbtcAuth +from .hitbtc_utils import HitbtcAPIError from .hitbtc_websocket import HitbtcWebsocket @@ -27,6 +28,7 @@ def logger(cls) -> HummingbotLogger: def __init__(self, hitbtc_auth: HitbtcAuth, trading_pairs: Optional[List[str]] = []): self._hitbtc_auth: HitbtcAuth = hitbtc_auth + self._ws_trade: HitbtcWebsocket = None self._trading_pairs = trading_pairs self._current_listen_key = None self._listen_for_user_stream_task = None @@ -43,13 +45,13 @@ async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: """ try: - ws = HitbtcWebsocket(self._hitbtc_auth) + self._ws_trade = HitbtcWebsocket(self._hitbtc_auth) - await ws.connect() + await self._ws_trade.connect() - await ws.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"], None, {}) + await self._ws_trade.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"], None, {}) - async for msg in ws.on_message(): + async for msg in self._ws_trade.on_message(): self._last_recv_time = time.time() if msg.get("params") is None: @@ -58,7 +60,7 @@ async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: except Exception as e: raise e finally: - await ws.disconnect() + await self._ws_trade.disconnect() await asyncio.sleep(5) async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue) -> AsyncIterable[Any]: @@ -75,6 +77,9 @@ async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: a output.put_nowait(msg) except asyncio.CancelledError: raise + except HitbtcAPIError as e: + self.logger().error(e.error_payload.get('error'), exc_info=True) + raise except Exception: self.logger().error( f"Unexpected error with {Constants.EXCHANGE_NAME} WebSocket connection. " diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index 94fc62e247..90952aed1f 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -573,19 +573,8 @@ async def _update_balances(self): """ Calls REST API to update total and available balances. """ - local_asset_names = set(self._account_balances.keys()) - remote_asset_names = set() account_info = await self._api_request("GET", Constants.ENDPOINT["USER_BALANCES"], is_auth_required=True) - for account in account_info: - asset_name = account["currency"] - self._account_available_balances[asset_name] = Decimal(str(account["available"])) - self._account_balances[asset_name] = Decimal(str(account["reserved"])) + Decimal(str(account["available"])) - remote_asset_names.add(asset_name) - - asset_names_to_remove = local_asset_names.difference(remote_asset_names) - for asset_name in asset_names_to_remove: - del self._account_available_balances[asset_name] - del self._account_balances[asset_name] + self._process_balance_message(account_info) async def _update_order_status(self): """ @@ -747,6 +736,20 @@ async def _process_trade_message(self, trade_msg: Dict[str, Any]): tracked_order.order_type)) self.stop_tracking_order(tracked_order.client_order_id) + def _process_balance_message(self, balance_update): + local_asset_names = set(self._account_balances.keys()) + remote_asset_names = set() + for account in balance_update: + asset_name = account["currency"] + self._account_available_balances[asset_name] = Decimal(str(account["available"])) + self._account_balances[asset_name] = Decimal(str(account["reserved"])) + Decimal(str(account["available"])) + remote_asset_names.add(asset_name) + + asset_names_to_remove = local_asset_names.difference(remote_asset_names) + for asset_name in asset_names_to_remove: + del self._account_available_balances[asset_name] + del self._account_balances[asset_name] + async def cancel_all(self, timeout_seconds: float) -> List[CancellationResult]: """ Cancels all in-flight orders and waits for cancellation results. diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py index a721fc0320..97a646f34f 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py @@ -16,7 +16,10 @@ from websockets.exceptions import ConnectionClosed from hummingbot.logger import HummingbotLogger from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth -from hummingbot.connector.exchange.hitbtc.hitbtc_utils import RequestId +from hummingbot.connector.exchange.hitbtc.hitbtc_utils import ( + RequestId, + HitbtcAPIError, +) # reusable websocket class # ToDo: We should eventually remove this class, and instantiate web socket connection normally (see Binance for example) @@ -50,7 +53,7 @@ async def connect(self): json_msg = json.loads(raw_msg_str) if json_msg.get("result") is not True: err_msg = json_msg.get('error', {}).get('message') - raise Exception(f"Failed to authenticate to websocket - {err_msg}.") + raise HitbtcAPIError({"error": f"Failed to authenticate to websocket - {err_msg}."}) return self._client @@ -68,10 +71,10 @@ async def _messages(self) -> AsyncIterable[Any]: try: raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=Constants.MESSAGE_TIMEOUT) try: - raw_msg = json.loads(raw_msg_str) + msg = json.loads(raw_msg_str) # HitBTC doesn't support ping or heartbeat messages. # Can handle them here if that changes - use `safe_ensure_future`. - yield raw_msg + yield msg except ValueError: continue except asyncio.TimeoutError: From 9aa88aa052a40dfc06c8bd958a9850b646108c91 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Fri, 19 Mar 2021 03:45:44 +0000 Subject: [PATCH 27/33] HitBTC: Add Balance WS Request (Disabled) --- .../hitbtc_api_user_stream_data_source.py | 30 ++++++++++++++++++- .../exchange/hitbtc/hitbtc_constants.py | 1 + .../exchange/hitbtc/hitbtc_exchange.py | 5 +++- .../exchange/hitbtc/hitbtc_websocket.py | 14 +++++++-- 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py index 77d09fa9a8..d8cf7fdcb2 100755 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py @@ -29,6 +29,7 @@ def logger(cls) -> HummingbotLogger: def __init__(self, hitbtc_auth: HitbtcAuth, trading_pairs: Optional[List[str]] = []): self._hitbtc_auth: HitbtcAuth = hitbtc_auth self._ws_trade: HitbtcWebsocket = None + self._ws_accts: HitbtcWebsocket = None self._trading_pairs = trading_pairs self._current_listen_key = None self._listen_for_user_stream_task = None @@ -39,23 +40,50 @@ def __init__(self, hitbtc_auth: HitbtcAuth, trading_pairs: Optional[List[str]] = def last_recv_time(self) -> float: return self._last_recv_time + # ---- UNCOMMENT FOR ACCT WS --------------------> + async def _ws_request_balances(self): + await self._ws_accts.request("getBalance") + balance_response = None + async for msg in self._ws_accts.on_message(): + balance_response = msg + break + return balance_response + # <-------------------- UNCOMMENT FOR ACCT WS ---- + async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: """ Subscribe to active orders via web socket """ try: + # ---- UNCOMMENT FOR ACCT WS --------------------> + # self._ws_accts = HitbtcWebsocket(self._hitbtc_auth, ws_acct=True) + # <-------------------- UNCOMMENT FOR ACCT WS ---- self._ws_trade = HitbtcWebsocket(self._hitbtc_auth) + # ---- UNCOMMENT FOR ACCT WS --------------------> + # await self._ws_accts.connect() + # <-------------------- UNCOMMENT FOR ACCT WS ---- await self._ws_trade.connect() await self._ws_trade.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"], None, {}) + # ---- UNCOMMENT FOR ACCT WS --------------------> + # event_methods = [ + # Constants.WS_METHODS["USER_ORDERS"], + # Constants.WS_METHODS["USER_TRADES"], + # ] + # <-------------------- UNCOMMENT FOR ACCT WS ---- + async for msg in self._ws_trade.on_message(): self._last_recv_time = time.time() - if msg.get("params") is None: + if msg.get("params", msg.get("result", None)) is None: continue + # ---- UNCOMMENT FOR ACCT WS --------------------> + # elif msg.get("method", None) in event_methods: + # yield await self._ws_request_balances() + # <-------------------- UNCOMMENT FOR ACCT WS ---- yield msg except Exception as e: raise e diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py index 3ac2200f7c..cd4888d64a 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py @@ -5,6 +5,7 @@ class Constants: REST_URL_AUTH = "/api/2" # WS_PRIVATE_URL = "wss://stream.crypto.com/v2/user" WS_PRIVATE_URL = "wss://api.hitbtc.com/api/2/ws/trading" + WS_PRIVATE_ACCT_URL = "wss://api.hitbtc.com/api/2/ws/account" # WS_PUBLIC_URL = "wss://stream.crypto.com/v2/market" WS_PUBLIC_URL = "wss://api.hitbtc.com/api/2/ws/public" diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index 90952aed1f..f91dc81a64 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -832,14 +832,17 @@ async def _user_stream_event_listener(self): ] method: str = event_message.get("method", None) params: str = event_message.get("params", None) + account_balances: list = event_message.get("result", None) - if params is None or method not in event_methods: + if method not in event_methods and account_balances is None: continue if method == Constants.WS_METHODS["USER_TRADES"]: await self._process_trade_message(params) elif method == Constants.WS_METHODS["USER_ORDERS"]: for order_msg in params: self._process_order_message(order_msg) + elif isinstance(account_balances, list) and "currency" in account_balances[0]: + self._process_balance_message(account_balances) except asyncio.CancelledError: raise except Exception: diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py index 97a646f34f..ccc81a9123 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py @@ -34,10 +34,18 @@ def logger(cls) -> HummingbotLogger: cls._logger = logging.getLogger(__name__) return cls._logger - def __init__(self, auth: Optional[HitbtcAuth] = None): + def __init__(self, + auth: Optional[HitbtcAuth] = None, + ws_acct: bool = False): self._auth: Optional[HitbtcAuth] = auth self._isPrivate = True if self._auth is not None else False - self._WS_URL = Constants.WS_PRIVATE_URL if self._isPrivate else Constants.WS_PUBLIC_URL + self._isAcct = ws_acct + if self._isAcct is True: + self._WS_URL = Constants.WS_PRIVATE_ACCT_URL + elif self._isPrivate: + self._WS_URL = Constants.WS_PRIVATE_URL + else: + self._WS_URL = Constants.WS_PUBLIC_URL self._client: Optional[websockets.WebSocketClientProtocol] = None # connect to exchange @@ -72,6 +80,8 @@ async def _messages(self) -> AsyncIterable[Any]: raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=Constants.MESSAGE_TIMEOUT) try: msg = json.loads(raw_msg_str) + if self._isAcct: + self.logger().network(f"Accounts WS Msg -->\n{msg}") # HitBTC doesn't support ping or heartbeat messages. # Can handle them here if that changes - use `safe_ensure_future`. yield msg From 067045fefd3e6e93d7ea4c7db327ad7c1e6b3e42 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Fri, 19 Mar 2021 14:12:54 +0000 Subject: [PATCH 28/33] HitBTC: Fix Trading Balance WS Request and Order Book --- .../hitbtc/hitbtc_active_order_tracker.pyx | 4 +- .../hitbtc_api_user_stream_data_source.py | 52 ++++++------------- .../exchange/hitbtc/hitbtc_constants.py | 4 +- .../exchange/hitbtc/hitbtc_exchange.py | 1 + .../exchange/hitbtc/hitbtc_websocket.py | 13 +---- 5 files changed, 23 insertions(+), 51 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx index ea918d239c..bef95bea6d 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx @@ -60,9 +60,9 @@ cdef class HitbtcActiveOrderTracker: double timestamp = message.timestamp double amount = 0 - if "bids" in content_keys: + if "bid" in content_keys: bid_entries = content["bid"] - if "asks" in content_keys: + if "ask" in content_keys: ask_entries = content["ask"] bids = s_empty_diff diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py index d8cf7fdcb2..954ab9c344 100755 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_api_user_stream_data_source.py @@ -28,8 +28,7 @@ def logger(cls) -> HummingbotLogger: def __init__(self, hitbtc_auth: HitbtcAuth, trading_pairs: Optional[List[str]] = []): self._hitbtc_auth: HitbtcAuth = hitbtc_auth - self._ws_trade: HitbtcWebsocket = None - self._ws_accts: HitbtcWebsocket = None + self._ws: HitbtcWebsocket = None self._trading_pairs = trading_pairs self._current_listen_key = None self._listen_for_user_stream_task = None @@ -40,15 +39,8 @@ def __init__(self, hitbtc_auth: HitbtcAuth, trading_pairs: Optional[List[str]] = def last_recv_time(self) -> float: return self._last_recv_time - # ---- UNCOMMENT FOR ACCT WS --------------------> async def _ws_request_balances(self): - await self._ws_accts.request("getBalance") - balance_response = None - async for msg in self._ws_accts.on_message(): - balance_response = msg - break - return balance_response - # <-------------------- UNCOMMENT FOR ACCT WS ---- + return await self._ws.request(Constants.WS_METHODS["USER_BALANCE"]) async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: """ @@ -56,39 +48,29 @@ async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: """ try: - # ---- UNCOMMENT FOR ACCT WS --------------------> - # self._ws_accts = HitbtcWebsocket(self._hitbtc_auth, ws_acct=True) - # <-------------------- UNCOMMENT FOR ACCT WS ---- - self._ws_trade = HitbtcWebsocket(self._hitbtc_auth) - - # ---- UNCOMMENT FOR ACCT WS --------------------> - # await self._ws_accts.connect() - # <-------------------- UNCOMMENT FOR ACCT WS ---- - await self._ws_trade.connect() - - await self._ws_trade.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"], None, {}) - - # ---- UNCOMMENT FOR ACCT WS --------------------> - # event_methods = [ - # Constants.WS_METHODS["USER_ORDERS"], - # Constants.WS_METHODS["USER_TRADES"], - # ] - # <-------------------- UNCOMMENT FOR ACCT WS ---- - - async for msg in self._ws_trade.on_message(): + self._ws = HitbtcWebsocket(self._hitbtc_auth) + + await self._ws.connect() + + await self._ws.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"], None, {}) + + event_methods = [ + Constants.WS_METHODS["USER_ORDERS"], + Constants.WS_METHODS["USER_TRADES"], + ] + + async for msg in self._ws.on_message(): self._last_recv_time = time.time() if msg.get("params", msg.get("result", None)) is None: continue - # ---- UNCOMMENT FOR ACCT WS --------------------> - # elif msg.get("method", None) in event_methods: - # yield await self._ws_request_balances() - # <-------------------- UNCOMMENT FOR ACCT WS ---- + elif msg.get("method", None) in event_methods: + await self._ws_request_balances() yield msg except Exception as e: raise e finally: - await self._ws_trade.disconnect() + await self._ws.disconnect() await asyncio.sleep(5) async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue) -> AsyncIterable[Any]: diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py index cd4888d64a..f3dfe4230e 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py @@ -3,10 +3,7 @@ class Constants: EXCHANGE_NAME = "hitbtc" REST_URL = "https://api.hitbtc.com/api/2" REST_URL_AUTH = "/api/2" - # WS_PRIVATE_URL = "wss://stream.crypto.com/v2/user" WS_PRIVATE_URL = "wss://api.hitbtc.com/api/2/ws/trading" - WS_PRIVATE_ACCT_URL = "wss://api.hitbtc.com/api/2/ws/account" - # WS_PUBLIC_URL = "wss://stream.crypto.com/v2/market" WS_PUBLIC_URL = "wss://api.hitbtc.com/api/2/ws/public" HBOT_BROKER_ID = "refzzz48" @@ -36,6 +33,7 @@ class Constants: "ORDERS_UPDATE": "updateOrderbook", "TRADES_SNAPSHOT": "snapshotTrades", "TRADES_UPDATE": "updateTrades", + "USER_BALANCE": "getTradingBalance", "USER_ORDERS": "activeOrders", "USER_TRADES": "report", } diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index f91dc81a64..9f6f83ec15 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -835,6 +835,7 @@ async def _user_stream_event_listener(self): account_balances: list = event_message.get("result", None) if method not in event_methods and account_balances is None: + self.logger().error(f"Unexpected message in user stream: {event_message}.", exc_info=True) continue if method == Constants.WS_METHODS["USER_TRADES"]: await self._process_trade_message(params) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py index ccc81a9123..da65b869a2 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_websocket.py @@ -35,17 +35,10 @@ def logger(cls) -> HummingbotLogger: return cls._logger def __init__(self, - auth: Optional[HitbtcAuth] = None, - ws_acct: bool = False): + auth: Optional[HitbtcAuth] = None): self._auth: Optional[HitbtcAuth] = auth self._isPrivate = True if self._auth is not None else False - self._isAcct = ws_acct - if self._isAcct is True: - self._WS_URL = Constants.WS_PRIVATE_ACCT_URL - elif self._isPrivate: - self._WS_URL = Constants.WS_PRIVATE_URL - else: - self._WS_URL = Constants.WS_PUBLIC_URL + self._WS_URL = Constants.WS_PRIVATE_URL if self._isPrivate else Constants.WS_PUBLIC_URL self._client: Optional[websockets.WebSocketClientProtocol] = None # connect to exchange @@ -80,8 +73,6 @@ async def _messages(self) -> AsyncIterable[Any]: raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=Constants.MESSAGE_TIMEOUT) try: msg = json.loads(raw_msg_str) - if self._isAcct: - self.logger().network(f"Accounts WS Msg -->\n{msg}") # HitBTC doesn't support ping or heartbeat messages. # Can handle them here if that changes - use `safe_ensure_future`. yield msg From fa7d2c0279e371f84587d3a141e045ec0129842d Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Fri, 19 Mar 2021 14:16:36 +0000 Subject: [PATCH 29/33] HitBTC: Adjust poll intervals now that we have WS balances --- hummingbot/connector/exchange/hitbtc/hitbtc_constants.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py index f3dfe4230e..538e0b21f2 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_constants.py @@ -47,9 +47,8 @@ class Constants: # Intervals # Only used when nothing is received from WS SHORT_POLL_INTERVAL = 5.0 - # HitBTC poll interval can't be too long since we don't get balances via websockets - LONG_POLL_INTERVAL = 20.0 - # One minute should be fine for order status since we get these via WS + # One minute should be fine since we get trades, orders and balances via WS + LONG_POLL_INTERVAL = 60.0 UPDATE_ORDER_STATUS_INTERVAL = 60.0 # 10 minute interval to update trading rules, these would likely never change whilst running. INTERVAL_TRADING_RULES = 600 From 016d5b8b56520d56a3a9764863e597d4863beede Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Mon, 22 Mar 2021 20:11:03 +0000 Subject: [PATCH 30/33] HitBTC: Fix unclosed aiohttp connection in tests --- test/connector/exchange/hitbtc/test_hitbtc_auth.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/test/connector/exchange/hitbtc/test_hitbtc_auth.py b/test/connector/exchange/hitbtc/test_hitbtc_auth.py index 6cc71c27e9..1943412ea3 100644 --- a/test/connector/exchange/hitbtc/test_hitbtc_auth.py +++ b/test/connector/exchange/hitbtc/test_hitbtc_auth.py @@ -27,7 +27,9 @@ def setUpClass(cls): async def rest_auth(self) -> Dict[Any, Any]: endpoint = Constants.ENDPOINT['USER_BALANCES'] headers = self.auth.get_headers("GET", f"{Constants.REST_URL_AUTH}/{endpoint}", None) - response = await aiohttp.ClientSession().get(f"{Constants.REST_URL}/{endpoint}", headers=headers) + http_client = aiohttp.ClientSession() + response = await http_client.get(f"{Constants.REST_URL}/{endpoint}", headers=headers) + await http_client.close() return await response.json() async def ws_auth(self) -> Dict[Any, Any]: @@ -44,12 +46,7 @@ def test_rest_auth(self): assert "currency" in result[0].keys() def test_ws_auth(self): - try: - response = self.ev_loop.run_until_complete(self.ws_auth()) - no_errors = True - except Exception: - no_errors = False - assert no_errors is True + response = self.ev_loop.run_until_complete(self.ws_auth()) if 'result' not in response: print(f"Unexpected response for API call: {response}") assert response['result'] is True From 569cf04a0c38f9a51d64f915abca8373d91e19de Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 24 Mar 2021 17:46:16 +0000 Subject: [PATCH 31/33] HitBTC: Add Logo, change status to green --- README.md | 2 +- assets/hitbtc_logo.png | Bin 0 -> 9639 bytes hummingbot/connector/connector_status.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 assets/hitbtc_logo.png diff --git a/README.md b/README.md index eb9e13553d..2efed8c4ca 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ We created hummingbot to promote **decentralized market-making**: enabling membe | Crypto.com | crypto_com | [Crypto.com](https://crypto.com/exchange) | 2 | [API](https://exchange-docs.crypto.com/#introduction) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | | DyDx | dydx | [dy/dx](https://dydx.exchange/) | 1 | [API](https://docs.dydx.exchange/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Eterbase | eterbase | [Eterbase](https://www.eterbase.com/) | * | [API](https://developers.eterbase.exchange/?version=latest) |![RED](https://via.placeholder.com/15/f03c15/?text=+) | -| HitBTC | hitbtc | [HitBTC](https://hitbtc.com/) | 2 | [API](https://api.hitbtc.com/) | ![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | +| HitBTC | hitbtc | [HitBTC](https://hitbtc.com/) | 2 | [API](https://api.hitbtc.com/) | ![GREEN](https://via.placeholder.com/15/008000/?text=+) | |Huobi Global| huobi | [Huobi Global](https://www.hbg.com) | 1 | [API](https://huobiapi.github.io/docs/spot/v1/en/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | KuCoin | kucoin | [KuCoin](https://www.kucoin.com/) | 1 | [API](https://docs.kucoin.com/#general) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Kraken | kraken | [Kraken](https://www.kraken.com/) | 1 | [API](https://www.kraken.com/features/api) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | diff --git a/assets/hitbtc_logo.png b/assets/hitbtc_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..efdac105174752b13b09cfd70766c20a3f6b6853 GIT binary patch literal 9639 zcmbt)Wmp`+vNkb7a1AcOgS$(xMHhFs1s0dZEf5?6!GpUkZovZt3%WSLgD$=}!9Q~D zIp=;q?zw;N{Fv$Kr|PZh>bL8u?wLqcWmybV5>zB4Bn){uX>}x|XXj6CHR{o!^sNbY32$-lCX3#2T{p8m|26=L1vcTE`uNuB%~K~U`<_!E>KCx!pVWn z?5_-)mxJ?@HWHGExRYyM5+IQEA-U<+s#f*^$!t-xV&=u_ z=tlD|1Zj|)g)7(@0(Nqw`ip30?&J;;rGE1CzlPx8{BKxCw||D|DPZhgX3p%KY#e`w z^e;gm@P8L|aQL^h8$=!S-~RrOz;2q}&LDPmkeidctHsmAS<(D8XDTjEHcl!=pqT~O@o&%jf2aU~Lh_Do5Hm*$ki4`g^%Dpi7;Gs7 z5a5&M;|B-|@W@DUa>__b@=HhnWF(~pqyc=Ad^}wL(v^0yaCZPXLjI*|`QN${|55iZ z8yuXUEK7r2!5$z>fUA=O)jvip1pd#y@c&1=f9hKPXI}*Vqb~cCGwgro_Wzsfe|0?( z=x_7i%zb+KH}^q~PlR`UVs^pF{X7y9HIclugr?WRLDm~Tvc4=t+fBOScibxszII(e@SP1j6_T%T?-ip8JuropYAAYh8I@W!l3ZW%KX5gF)$L1gk=jE%A z;`j!#x7d*SRu#{4B|0@F>)t`~X1MNILH2~)RQe;K|E3!Eve8BL-s+>tV^x(E(LYw^ z|K$cidP$Fj)HQv4F8iCvn3g5bk5od)ibrOu2Jm4C_%l<`&7Lo#YCr69U7)J6k`*5n zWP;STyt1@p|7!=uEanQaS`B3~Oe2QO=eGjPnaxnf*7PFAEC|Zp8>Oh9*eIyE8NI*l zeg9KGX-eUx}q0 zNEY1|jL(Gq=KWi(9*RxDZLn}|D`uz=W>MiZN;yIJ*j?P*e41h;by_}w?gW$qe1ANq zc%dc&miGo-XVMX%!3auN8h~G84wZXdgv{Mz-pmr`(`-v8DVH|avX~3TK3G9+$k~YF zeHS$xDj#OAdcr*if*(%Er_Q!ng+8Zyi}jouRLoLiDC}d(X5;9xCGp7P_DYcL`}BgrD%pNu3J$A%mJ76Im@1_{u@*{ z)9gnq7~FRuH3 z6DX?de{$hpeOGtbX`DylN2Z$)I%cV!!lGAaHA+(8yX-z^WS!&hky0|oLd>gPR@45% zPG2NHg`a2XelfJ^A^M9RbUcKc^hQIDh9x*TF+JsOC+$XMZ@txO-29eN~(I ziN!H1IVFY73wUYnGh9CYtiiWqD@i*i!Y^9YU4QqrHZ$5IuwIJ$dnVwnz`#GMeyKvv*j zotzGJJ9@NO&g`Q)>SB~RipsYXDNsu&Jh_<$to#hF!YzJi?rmH6$f~Mv~M8UhER7W4i@}A#M$Wns7A-*{*DzP3}$(!1^ z4z(O06q%s?l`n?PSEC{mtpXqC^j)+T(6+^iEGCR6Z0a_JBii1p;cxX3nkdntV_EI8 zv3K6pl}&LKw_HSMLCqRX+RX1fdeGr5go29jyWjl!?lgmlulL*n6uV51vm2AKpWDmm zC4!nWaJ96=AO38V-)@mW#OmIi+IMGS^3|ZOf$Q0} z(BkF=i8n0UK(Ms7Nv(C1Y@chObM2=uMvb)r6jiKXp*<4l!<-SoN6R?VwZgA|W(x-W z7h5}mR=$hH)J7^{#OYhkr}neRV`GL#M@3Qi9&k1--d8&L5ATXBPraS6IWH^8_Sr2; zeIC>m(O%SBfo^z>yh+SG&mvy7Z?C2|(eNs}%8aE9b{axBiNG4#ly1l*uC$e^?6zt! ze$MdaToSFHMMK!mY({*k=w}&D#)TXj>||Ged=D>}hq$aP=&;q)Lie)%Sy&aPF1Lqz z_fYuqUH<@{tcRzs-#HU^YTc*i4DX-S;Es+CmfS<^|7VYl3Z@5cw~YMhJf)h{|s^WYf|D!+ig&h0j$Q@Oz(bhJ}SiMWG56 z=1TZjTU&d2Ho@Qp*&iR|ll{qPEx9O}PlB86Ud-RJkvkS7-oF$&ak^=xyi1JGF80e8 zb1M@%%Y=k{?attV*r$#E%<=;D5*F;moX0Xg-C5Q0~VcZ}H1ZI;GsTl;y+Pe>5;V){O`$%3Ec9 z!KpwQ_Pb&@SnJFi^QYaL(<0XB*`>V?=zuYOM9$D6Wo%kolY_^PiN&qLxpkoAz5a%~zE1=WW*e@X+h1;RgPWQZQrcuHzwm$_ zoHr2{#Nk|^M*LLoMVNW*V>^h;NUGcw2fYi&Mz}{h2v-j1Cr3L?y<)cDaFERSz+8qS zBO!cfpsTg#-{HubEVAOUvl)acly{6S*wRvh6LjeL|cGuYE#-qDeMOS zbe=GlG%7)fK3x0|YA@;Er}bFMI1il4e@Na6iH)0&Go}%J-I$a#c`hB5BB@M^`yvSK zaI-Wg00e(3A7^YE7ZXx!8iEFQJ@79LZ0I6A&< z<@`kG=}kRPbf#c>Jz;hRlQ{+fR$}^9im7SS+^}b50VKCJ%;~egR!HZ>Gt7rdhbGOe(O{lFk{LzXt z2~%zGLoGQqKdId=YnU4;irvD5#Xtgy;MUOu;lk!LXJi#tAl0vi^}vFtNG0lY@7qXi zzPH$qH^BkFJ3;bMw1uhyN^PQtn+@eT#ib26GHQjmCWeNFdyuyb`m9E`y*F0vkmT9s zkKF=U@svcR6DdNc?-u@)N5CvRK(AdWb#Qa`tntT^=XtLEz8Y3rWXin{BU|ojE@_2A z!BO>_sY-xL1&;$P76-O^C0}FwXQ5wh!9d&3s6{-hbKAH_j9@oRFquR z^JwE?zsc3aS0L^5`Yun^=Ih%6cd}-lA8R!O8!kdw$;n!vAu+pEPRH)hz4>y8!5aq+ z$&1ISt2AKeo16Bms%#ey1X8a*%a*_+nxOmM^rBN|);V|0D~qMc$%O7a?Mi)K&`?tM z?<56^`@79V6$$}o({nCvMp2#q9mX)^FvqxoAA5aCClG@snSm7h0cqvu{Jn6H+H+y2B z3(QNKX2v>@N)zE|4L*husYxx2xZ)BXYgiN!Xo0Lv(y|}Emerv-2!G|LL_>aI)N!QB z@zq~bx`Q{_3;KIzX2zXJc;%1aTmv>@e*MCy&9Dlc;_j(*9FeVB{megvSdmiNrqEKD z@ZtpVih)%lZn33Mzo7vIt`yYuV3vIHfSObP@QHpC@1>)v>=iFRT1x)}(EOIGe8P0@ zu7^oAJO0t0$7f}8yk2r5-{z4Ol=R^ZYnntok+>wWYGZ9#fuNz_pk>{+uWuq^(dr*j zSfB#8sI2v}%(~f%VJ)5k8}5yskSp3mnoevC@|S;}qJr?qV8#6$ThGuWu{4TGesm^M z4TaY3hl0ZC$ANk$oKZ^-nBfj%g000QwzQbc>M9xeYwd?Jf}#Oi4fB#yhBBklJU9by zC_6~R&Rsln_z8!^cl!{l2Cngz zg2gxsHCP7vrPSfi2ncQTi_Isld)F+vh?KD>mXeTR(A@!$#>9jiu?X-mPEz*vredk4-GHs98LG z5ggF{iqL+cAyVNE4$oIjVKw;XvH`dWgb5?spQV~M>Z5r#bKmj29H;{&&)~rX0tdzK zTi0R-2v)3Eo&({Qn(nX^Ii7^3$?U0}!U~Uk0T-bO%^u&$n&m}`N3UpMY_3~%Ig4FG zwEBg=g@#VGan{zdB=lfwYik|_m=b?GPNNDcIqfj(342Z9WqNhA3{g~69KOF=4eTS( z$t1xhu-%MJ#&5eTer0VaII>k$8WbIsRaya%YNu8VBgDjG@q@372OUNx>3>D`DHI?Iv`P~3FE_o5gQ(2>?1Bhv0;>J`*X8qNEnhS}N!3gvQ;DRtUhWayG0MffO_w$jM& zV*I@I6*mr&V+-9w%wBRv##7%3Wa(t19l;1bu{F1=o9ErZKo;HTthd&JoB_f~777aRh~0xdcV_Pz>}?J|kn$4NKBIi~Hya+-PuL~z z>tM<1nmejgM@V0=bP|3-6OJd!H+$yp0s?(KRj4WWrCQ`;&$B%7?edqs#>jChitqeM z3N1e@Ju^L6^U8y3c$`fs3;x!9%<~_+b-s^XtmWN~7HzrjebJ z*39CVef8!f@`D1=3vX#W|*jXZUH^B%OE3-4;q*mrcz8GQ(~|JR|1MkGIv8Jh~b zKRJzH8I7f$>->Q?Tp;IhaE(73_P8X8$#7=rWBH>d_vr67AaYz8 z1jgyGJ*fuJ+1!CzrlsGlof(<-pDNN%=?N|)Ho>%_+B5hbRqYlOvtj|;CD$y7mgprH z1xa0tNkj!*7;3Ledu8cMwvK}!Qs_rBf}h4Xb;7~Kjz4gXlGlrvJB9BP>`=K~Wo@H0 zaQhddYJwS4w7UsjuVdn8XP7CSj+a!G6REWCnlSBWxUx=o6SZ*H_0ku_Owh@_=?;E1 z!y_sF+T}HBY=tDy*i^c_ma=>3*gVjuyBPvSWy|2W=x=lLOWP`_*T7Yyqp;HLDYv!A zN>a{xalL-RG1tR+ zW7U3T`gNu8&|)^e18<(>ES>>lL)_0Yd9dK-7hN2`y&k%Jn=cL_B4T%0L?JKHG{k~N z456cKf4C$0v9qzmF0J+8_#{~b&sLR+)X(6sr}PWr8f&RX8BCb=IR6@^ zD^)M;mF+1&MnFK2Jmg4vP0-?l@wNi_l$)raE`8`4{&JZep2g|LX8@RSADq!9L*wX6 zj_D>WG?ihqZE0>ejjvgKO-|9fYzouRYFc!tpRW^)U>Tk&Q3)5bO!?_=YNAB6{98Q~ z5&>8g!i|(>W+puu%kMz^uCKC6T4Pv^?5_d2rPc&`3-ZJ4wl=0!vp8Hj+xnOLlax7_ zj?lH`w>EFkbV5!1o(qXYv~kkPqh2L$xtMe6QV&_G1@=qI(XDJ;)V)->-Sl^y01?&p zn7~qOpd8>>s^dE7Rf}kiDo*nB%9H}@scK+EVwXDqOpEJ)AiEe?(<^I;o5=#7^rlquWnS{pTTkXTS zb5FUTtJEeXY)XZXSl*_vWEth{*cwaePop_`TCzzjdkcBM6-h)DN9n_CIMjON6?url zvkynr#0K%&>fi$~{f`}%f%z!ZQEg87*uz_9I6^_3G&88ZJjz&I^tOtDX^+%R=V$A+ zV#>qs_)NOdr_|U*o5YvXji1aJrp=_RazSsnwiTyQhkM{wd)t(+q;oQ}MO&36hWIY7 z3Ck2;?<>III{cB$Jtxv*ayRrUNV23I zi_zStiLVX&?0z+=Ya!@Zk{2CTLz%Y^a-m)?hqcx9rd$U)`Ik61 zH*9pRt4;O}LzYU3V3qR?Wj$T}sp*2=4bh1%_N;7rF1NvG+6-5ZyniNy zzxyw4eaN9@Hlg3aESb_5EpuJb>pL6s7!fjHuj+;7(|0zLUuXuGl{!7Z-Pa2qe>J*2 z?aNImf|!3BeyRGoWB15$*uEdrcfbJgs5g{Vv-bw5+Ey%kU$;gnQQ!F6k(Epi_9>Et6pc!N;-2cwg=| z%eMn+jF|~?PjKwr)M*H2Q!FgB-4lgH#jVa?u4Fu&;^}eWy~;wPZgrgQ*ww!Q!Su3I zg-{wxfbvDCv!@ffBZnvRO%D>8;}t2IT%KeSZ-XBnxll@6oTY1Q#tw_W`*D5JI zX5|CNctFHr-CG>ddVT}{$f14dWEZ9&x*I&o)5I0Y9%tloC1e#fFeH~@diBBPY{)=~ z1p8^FS|z?Y!-#t|wurMYQfl7ug_JE>p`|_OS9WE}VIXxros$kw;F&fktE_Fy1io_o zI8utBI-2jNF%41cholml3*L9#;&kf3I#;-cz^%>#OZz9xi^g>UHx#uCz;Q3#L{QoK z0_DhK|8nNM5Jf~=S`|i@pR)+zg}m=-oV+VViepj%o{vx*41VJEv$&-9a=F(W*?-QR zH<_v9XyuxJ_0>}f>TP8EWJV1Xk(h{^G_&)Yq)kT|cLG8<57L2vJ6+JVgJG+vk6yF*UdvjMJ=+QW}0==E=K%4ZmZQ*Va&o0Za8*llB-}Ry(DlsVS-EXCc;F|bK-Wv?KkquW(R(&~- zQ+0Rr?hk#mqVJ4`bKY8db=L)Q*I&=)&MnVO5uziH@&-NcCFdZfSFL;4a_AdXQ50VB z@V=k0sY*@a+)t`(_>LFsu6}*##PYzlm1*ZbF;?pB4A-ln(!ykAa@<}Y%0j`+qM=u=ILv~{cq*~AZLMh7WKDP&VkTbjFC%mhMpj? zz?$l6l@w4S$=gY;-=-hq-&{3&efV|8RKH&-{cCj}Xvh6#hh_5J=DtlrxQ_=6{GU64ODR5KD<2I*eUzIj~ZF9w<8S=4mhfaauJ@ znKO%RwLOugoxMzn9dpu}Q{F%73`KsD-+Wx`ltJ8RrfB4O{Wv@U{8&3V zQGFia+uiptA4k-`*Us=mvlk~t%6`m;T}Vt{MbYF@B`_j7BDRoGAw)3n^`3B)VNSwF zwBfQsXgIZ64vh3OV1RlNF!>Os-;*nWGOXc0++>^;aA?ax6x}y8Gz4X0&WhAlgfp1S zWQl<|!TK@fq*J?p9uSs=1@+D@?RqyBq zLzftT^|fMh9`T8878?uZt;wvmw$^3sN2q2y`g{8O@5SKC%1TjD(E>exeHRk!!-d9Z0Dw3r)u;Fs+S~CcQZ8Xp zcUjqiY*F7j%3~o#g>4r5+SH^k8feGY=g)6O$TQEcldALHk;VlwWUbJ}ghf_yrYqC{ zCZ}rMeE?bvbx`>-O_8_J47k(saV?DFrl6Jll=Vq2{jPTx;RrgiF@?Q z37wF|5X%iUWoR=rw8P(rDW&ahpi4Cd+5)xJ*PjPtSF4eitkE`7Bum~{KNNz%X4C;# z1#-ma@ET(Jka1j0lv*m3h91HTi+Xz2H4enl?-mF&wC3p(ci7f1n5#|5&NTnjSJC^C z@-TOT{knYhY9;n(KJ%Cj^xRdPE}P1qB+6$L{QnRRoxcwjJ4ZXOXBnoMY|xwq6Whpt zKW!B%Le~^41z5TnVQ4>|TZ{BfslLl~hj9g&*6K>Dkty@4oB)yz$eAhySdkj Date: Thu, 25 Mar 2021 01:37:55 +0000 Subject: [PATCH 32/33] HitBTC: Change status to yellow --- README.md | 2 +- hummingbot/connector/connector_status.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2efed8c4ca..eb9e13553d 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ We created hummingbot to promote **decentralized market-making**: enabling membe | Crypto.com | crypto_com | [Crypto.com](https://crypto.com/exchange) | 2 | [API](https://exchange-docs.crypto.com/#introduction) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | | DyDx | dydx | [dy/dx](https://dydx.exchange/) | 1 | [API](https://docs.dydx.exchange/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Eterbase | eterbase | [Eterbase](https://www.eterbase.com/) | * | [API](https://developers.eterbase.exchange/?version=latest) |![RED](https://via.placeholder.com/15/f03c15/?text=+) | -| HitBTC | hitbtc | [HitBTC](https://hitbtc.com/) | 2 | [API](https://api.hitbtc.com/) | ![GREEN](https://via.placeholder.com/15/008000/?text=+) | +| HitBTC | hitbtc | [HitBTC](https://hitbtc.com/) | 2 | [API](https://api.hitbtc.com/) | ![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | |Huobi Global| huobi | [Huobi Global](https://www.hbg.com) | 1 | [API](https://huobiapi.github.io/docs/spot/v1/en/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | KuCoin | kucoin | [KuCoin](https://www.kucoin.com/) | 1 | [API](https://docs.kucoin.com/#general) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Kraken | kraken | [Kraken](https://www.kraken.com/) | 1 | [API](https://www.kraken.com/features/api) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | diff --git a/hummingbot/connector/connector_status.py b/hummingbot/connector/connector_status.py index 4fe990d105..2f35999bb7 100644 --- a/hummingbot/connector/connector_status.py +++ b/hummingbot/connector/connector_status.py @@ -17,7 +17,7 @@ 'dydx': 'green', 'eterbase': 'red', 'ethereum': 'red', - 'hitbtc': 'green', + 'hitbtc': 'yellow', 'huobi': 'green', 'kraken': 'green', 'kucoin': 'green', From e419bd859cfe043ac4278cef9ff37e7e949befee Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Thu, 25 Mar 2021 02:19:10 +0000 Subject: [PATCH 33/33] HitBTC: Remove unused code in order book --- .../hitbtc/hitbtc_active_order_tracker.pxd | 3 ++- .../hitbtc/hitbtc_active_order_tracker.pyx | 20 +++++++++---------- .../hitbtc/hitbtc_order_book_message.py | 18 ++++++++--------- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pxd b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pxd index 97c2af5b02..5babac5332 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pxd +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pxd @@ -7,4 +7,5 @@ cdef class HitbtcActiveOrderTracker: cdef tuple c_convert_diff_message_to_np_arrays(self, object message) cdef tuple c_convert_snapshot_message_to_np_arrays(self, object message) - cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message) + # This method doesn't seem to be used anywhere at all + # cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx index bef95bea6d..5e248bb3d5 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_active_order_tracker.pyx @@ -132,17 +132,15 @@ cdef class HitbtcActiveOrderTracker: return bids, asks - cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message): - cdef: - double trade_type_value = 1.0 if message.content["side"] == "buy" else 2.0 - - timestamp = message.timestamp - content = message.content - - return np.array( - [timestamp, trade_type_value, float(content["price"]), float(content["quantity"])], - dtype="float64" - ) + # This method doesn't seem to be used anywhere at all + # cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message): + # cdef: + # double trade_type_value = 1.0 if message.content["side"] == "buy" else 2.0 + # list content = message.content + # return np.array( + # [message.timestamp, trade_type_value, float(content["price"]), float(content["quantity"])], + # dtype="float64" + # ) def convert_diff_message_to_order_book_row(self, message): np_bids, np_asks = self.c_convert_diff_message_to_np_arrays(message) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py index 1f0bc1d631..fdc207d64d 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_order_book_message.py @@ -11,6 +11,7 @@ OrderBookMessage, OrderBookMessageType, ) +from .hitbtc_constants import Constants from .hitbtc_utils import ( convert_from_exchange_trading_pair, ) @@ -54,21 +55,18 @@ def trading_pair(self) -> str: elif "symbol" in self.content: return convert_from_exchange_trading_pair(self.content["symbol"]) + # The `asks` and `bids` properties are only used in the methods below. + # They are all replaced or unused in this connector: + # OrderBook.restore_from_snapshot_and_diffs + # OrderBookTracker._track_single_book + # MockAPIOrderBookDataSource.get_tracking_pairs @property def asks(self) -> List[OrderBookRow]: - asks = map(self.content["ask"], lambda ask: {"price": ask["price"], "size": ask["size"]}) - - return [ - OrderBookRow(float(price), float(amount), self.update_id) for price, amount in asks - ] + raise NotImplementedError(Constants.EXCHANGE_NAME + " order book uses active_order_tracker.") @property def bids(self) -> List[OrderBookRow]: - bids = map(self.content["bid"], lambda bid: {"price": bid["price"], "size": bid["size"]}) - - return [ - OrderBookRow(float(price), float(amount), self.update_id) for price, amount in bids - ] + raise NotImplementedError(Constants.EXCHANGE_NAME + " order book uses active_order_tracker.") def __eq__(self, other) -> bool: return self.type == other.type and self.timestamp == other.timestamp