From 0c929264aa4b9a7921e32f3bc05d5634cbc7ee86 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Sat, 20 Mar 2021 03:02:28 +0000 Subject: [PATCH 01/23] CoinZoom: Initial Template Clone --- README.md | 1 + conf/__init__.py | 4 + hummingbot/connector/connector_status.py | 1 + .../connector/exchange/coinzoom/__init__.py | 0 .../coinzoom_active_order_tracker.pxd | 10 + .../coinzoom_active_order_tracker.pyx | 157 ++++ .../coinzoom_api_order_book_data_source.py | 216 +++++ .../coinzoom_api_user_stream_data_source.py | 97 ++ .../exchange/coinzoom/coinzoom_auth.py | 72 ++ .../exchange/coinzoom/coinzoom_constants.py | 57 ++ .../exchange/coinzoom/coinzoom_exchange.py | 877 ++++++++++++++++++ .../coinzoom/coinzoom_in_flight_order.py | 118 +++ .../exchange/coinzoom/coinzoom_order_book.py | 146 +++ .../coinzoom/coinzoom_order_book_message.py | 83 ++ .../coinzoom/coinzoom_order_book_tracker.py | 109 +++ .../coinzoom_order_book_tracker_entry.py | 21 + .../coinzoom/coinzoom_user_stream_tracker.py | 73 ++ .../exchange/coinzoom/coinzoom_utils.py | 156 ++++ .../exchange/coinzoom/coinzoom_websocket.py | 130 +++ .../templates/conf_fee_overrides_TEMPLATE.yml | 3 + hummingbot/templates/conf_global_TEMPLATE.yml | 3 + setup.py | 1 + test/connector/exchange/coinzoom/.gitignore | 1 + test/connector/exchange/coinzoom/__init__.py | 0 .../exchange/coinzoom/test_coinzoom_auth.py | 55 ++ .../coinzoom/test_coinzoom_exchange.py | 438 +++++++++ .../test_coinzoom_order_book_tracker.py | 103 ++ .../test_coinzoom_user_stream_tracker.py | 37 + 28 files changed, 2969 insertions(+) create mode 100644 hummingbot/connector/exchange/coinzoom/__init__.py create mode 100644 hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd create mode 100644 hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx create mode 100644 hummingbot/connector/exchange/coinzoom/coinzoom_api_order_book_data_source.py create mode 100755 hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py create mode 100755 hummingbot/connector/exchange/coinzoom/coinzoom_auth.py create mode 100644 hummingbot/connector/exchange/coinzoom/coinzoom_constants.py create mode 100644 hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py create mode 100644 hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py create mode 100644 hummingbot/connector/exchange/coinzoom/coinzoom_order_book.py create mode 100644 hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py create mode 100644 hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker.py create mode 100644 hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker_entry.py create mode 100644 hummingbot/connector/exchange/coinzoom/coinzoom_user_stream_tracker.py create mode 100644 hummingbot/connector/exchange/coinzoom/coinzoom_utils.py create mode 100644 hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py create mode 100644 test/connector/exchange/coinzoom/.gitignore create mode 100644 test/connector/exchange/coinzoom/__init__.py create mode 100644 test/connector/exchange/coinzoom/test_coinzoom_auth.py create mode 100644 test/connector/exchange/coinzoom/test_coinzoom_exchange.py create mode 100755 test/connector/exchange/coinzoom/test_coinzoom_order_book_tracker.py create mode 100644 test/connector/exchange/coinzoom/test_coinzoom_user_stream_tracker.py diff --git a/README.md b/README.md index eb9e13553d..8c94efb3ea 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ We created hummingbot to promote **decentralized market-making**: enabling membe | BitMax | bitmax | [BitMax](https://bitmax.io/en/global-digital-asset-platform) | 1 | [API](https://bitmax-exchange.github.io/bitmax-pro-api/#bitmax-pro-api-documentation) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Blocktane | blocktane | [Blocktane](https://blocktane.io/) | 2 | [API](https://blocktane.io/api) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Coinbase Pro | coinbase_pro | [Coinbase Pro](https://pro.coinbase.com/) | * | [API](https://docs.pro.coinbase.com/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | +| CoinZoom | coinzoom | [CoinZoom](https://trade.coinzoom.com/landing) | * | [API](https://api-docs.coinzoom.com/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Crypto.com | crypto_com | [Crypto.com](https://crypto.com/exchange) | 2 | [API](https://exchange-docs.crypto.com/#introduction) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | | DyDx | dydx | [dy/dx](https://dydx.exchange/) | 1 | [API](https://docs.dydx.exchange/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Eterbase | eterbase | [Eterbase](https://www.eterbase.com/) | * | [API](https://developers.eterbase.exchange/?version=latest) |![RED](https://via.placeholder.com/15/f03c15/?text=+) | diff --git a/conf/__init__.py b/conf/__init__.py index 7854c51a99..f096f7f933 100644 --- a/conf/__init__.py +++ b/conf/__init__.py @@ -108,6 +108,10 @@ hitbtc_api_key = os.getenv("HITBTC_API_KEY") hitbtc_secret_key = os.getenv("HITBTC_SECRET_KEY") +# CoinZoom Test +coinzoom_api_key = os.getenv("COINZOOM_API_KEY") +coinzoom_secret_key = os.getenv("COINZOOM_SECRET_KEY") + # Wallet Tests test_erc20_token_address = os.getenv("TEST_ERC20_TOKEN_ADDRESS") web3_test_private_key_a = os.getenv("TEST_WALLET_PRIVATE_KEY_A") diff --git a/hummingbot/connector/connector_status.py b/hummingbot/connector/connector_status.py index 2f35999bb7..0acc21b37d 100644 --- a/hummingbot/connector/connector_status.py +++ b/hummingbot/connector/connector_status.py @@ -13,6 +13,7 @@ 'blocktane': 'green', 'celo': 'green', 'coinbase_pro': 'green', + 'coinzoom': 'yellow', 'crypto_com': 'yellow', 'dydx': 'green', 'eterbase': 'red', diff --git a/hummingbot/connector/exchange/coinzoom/__init__.py b/hummingbot/connector/exchange/coinzoom/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd new file mode 100644 index 0000000000..97c2af5b02 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd @@ -0,0 +1,10 @@ +# distutils: language=c++ +cimport numpy as np + +cdef class HitbtcActiveOrderTracker: + cdef dict _active_bids + cdef dict _active_asks + + cdef tuple c_convert_diff_message_to_np_arrays(self, object message) + cdef tuple c_convert_snapshot_message_to_np_arrays(self, object message) + cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx new file mode 100644 index 0000000000..bef95bea6d --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx @@ -0,0 +1,157 @@ +# distutils: language=c++ +# distutils: sources=hummingbot/core/cpp/OrderBookEntry.cpp +import logging +import numpy as np +from decimal import Decimal +from typing import Dict +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.order_book_row import OrderBookRow + +_logger = None +s_empty_diff = np.ndarray(shape=(0, 4), dtype="float64") +HitbtcOrderBookTrackingDictionary = Dict[Decimal, Dict[str, Dict[str, any]]] + +cdef class HitbtcActiveOrderTracker: + def __init__(self, + active_asks: HitbtcOrderBookTrackingDictionary = None, + active_bids: HitbtcOrderBookTrackingDictionary = None): + super().__init__() + self._active_asks = active_asks or {} + self._active_bids = active_bids or {} + + @classmethod + def logger(cls) -> HummingbotLogger: + global _logger + if _logger is None: + _logger = logging.getLogger(__name__) + return _logger + + @property + def active_asks(self) -> HitbtcOrderBookTrackingDictionary: + return self._active_asks + + @property + def active_bids(self) -> HitbtcOrderBookTrackingDictionary: + return self._active_bids + + # TODO: research this more + def volume_for_ask_price(self, price) -> float: + return NotImplementedError + + # TODO: research this more + def volume_for_bid_price(self, price) -> float: + return NotImplementedError + + def get_rates_and_quantities(self, entry) -> tuple: + # price, quantity + return float(entry["price"]), float(entry["size"]) + + cdef tuple c_convert_diff_message_to_np_arrays(self, object message): + cdef: + dict content = message.content + list content_keys = list(content.keys()) + list bid_entries = [] + list ask_entries = [] + str order_id + str order_side + str price_raw + object price + dict order_dict + double timestamp = message.timestamp + double amount = 0 + + if "bid" in content_keys: + bid_entries = content["bid"] + if "ask" in content_keys: + ask_entries = content["ask"] + + bids = s_empty_diff + asks = s_empty_diff + + if len(bid_entries) > 0: + bids = np.array( + [[timestamp, + price, + amount, + message.update_id] + for price, amount in [self.get_rates_and_quantities(entry) for entry in bid_entries]], + dtype="float64", + ndmin=2 + ) + + if len(ask_entries) > 0: + asks = np.array( + [[timestamp, + price, + amount, + message.update_id] + for price, amount in [self.get_rates_and_quantities(entry) for entry in ask_entries]], + dtype="float64", + ndmin=2 + ) + + return bids, asks + + cdef tuple c_convert_snapshot_message_to_np_arrays(self, object message): + cdef: + float price + float amount + str order_id + dict order_dict + + # Refresh all order tracking. + self._active_bids.clear() + self._active_asks.clear() + timestamp = message.timestamp + content = message.content + + for snapshot_orders, active_orders in [(content["bid"], self._active_bids), (content["ask"], self._active_asks)]: + for entry in snapshot_orders: + price, amount = self.get_rates_and_quantities(entry) + active_orders[price] = amount + + # Return the sorted snapshot tables. + cdef: + np.ndarray[np.float64_t, ndim=2] bids = np.array( + [[message.timestamp, + float(price), + float(self._active_bids[price]), + message.update_id] + for price in sorted(self._active_bids.keys())], dtype='float64', ndmin=2) + np.ndarray[np.float64_t, ndim=2] asks = np.array( + [[message.timestamp, + float(price), + float(self._active_asks[price]), + message.update_id] + for price in sorted(self._active_asks.keys(), reverse=True)], dtype='float64', ndmin=2) + + if bids.shape[1] != 4: + bids = bids.reshape((0, 4)) + if asks.shape[1] != 4: + asks = asks.reshape((0, 4)) + + return bids, asks + + cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message): + cdef: + double trade_type_value = 1.0 if message.content["side"] == "buy" else 2.0 + + timestamp = message.timestamp + content = message.content + + return np.array( + [timestamp, trade_type_value, float(content["price"]), float(content["quantity"])], + dtype="float64" + ) + + def convert_diff_message_to_order_book_row(self, message): + np_bids, np_asks = self.c_convert_diff_message_to_np_arrays(message) + bids_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_bids] + asks_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_asks] + return bids_row, asks_row + + def convert_snapshot_message_to_order_book_row(self, message): + np_bids, np_asks = self.c_convert_snapshot_message_to_np_arrays(message) + bids_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_bids] + asks_row = [OrderBookRow(price, qty, update_id) for ts, price, qty, update_id in np_asks] + return bids_row, asks_row diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_api_order_book_data_source.py b/hummingbot/connector/exchange/coinzoom/coinzoom_api_order_book_data_source.py new file mode 100644 index 0000000000..40d83516da --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_api_order_book_data_source.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python +import asyncio +import logging +import time +import pandas as pd +from decimal import Decimal +from typing import Optional, List, Dict, Any +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_message import OrderBookMessage +from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource +from hummingbot.logger import HummingbotLogger +from .hitbtc_constants import Constants +from .hitbtc_active_order_tracker import HitbtcActiveOrderTracker +from .hitbtc_order_book import HitbtcOrderBook +from .hitbtc_websocket import HitbtcWebsocket +from .hitbtc_utils import ( + str_date_to_ts, + convert_to_exchange_trading_pair, + convert_from_exchange_trading_pair, + api_call_with_retries, + HitbtcAPIError, +) + + +class HitbtcAPIOrderBookDataSource(OrderBookTrackerDataSource): + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, trading_pairs: List[str] = None): + super().__init__(trading_pairs) + self._trading_pairs: List[str] = trading_pairs + self._snapshot_msg: Dict[str, any] = {} + + @classmethod + async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, Decimal]: + results = {} + if len(trading_pairs) > 1: + tickers: List[Dict[Any]] = await api_call_with_retries("GET", Constants.ENDPOINT["TICKER"]) + for trading_pair in trading_pairs: + ex_pair: str = convert_to_exchange_trading_pair(trading_pair) + if len(trading_pairs) > 1: + ticker: Dict[Any] = list([tic for tic in tickers if tic['symbol'] == ex_pair])[0] + else: + url_endpoint = Constants.ENDPOINT["TICKER_SINGLE"].format(trading_pair=ex_pair) + ticker: Dict[Any] = await api_call_with_retries("GET", url_endpoint) + results[trading_pair]: Decimal = Decimal(str(ticker["last"])) + return results + + @staticmethod + async def fetch_trading_pairs() -> List[str]: + try: + symbols: List[Dict[str, Any]] = await api_call_with_retries("GET", Constants.ENDPOINT["SYMBOL"]) + trading_pairs: List[str] = list([convert_from_exchange_trading_pair(sym["id"]) for sym in symbols]) + # Filter out unmatched pairs so nothing breaks + return [sym for sym in trading_pairs if sym is not None] + except Exception: + # Do nothing if the request fails -- there will be no autocomplete for HitBTC trading pairs + pass + return [] + + @staticmethod + async def get_order_book_data(trading_pair: str) -> Dict[str, any]: + """ + Get whole orderbook + """ + try: + ex_pair = convert_to_exchange_trading_pair(trading_pair) + orderbook_response: Dict[Any] = await api_call_with_retries("GET", Constants.ENDPOINT["ORDER_BOOK"], + params={"limit": 150, "symbols": ex_pair}) + return orderbook_response[ex_pair] + except HitbtcAPIError as e: + err = e.error_payload.get('error', e.error_payload) + raise IOError( + f"Error fetching OrderBook for {trading_pair} at {Constants.EXCHANGE_NAME}. " + f"HTTP status is {e.error_payload['status']}. Error is {err.get('message', str(err))}.") + + async def get_new_order_book(self, trading_pair: str) -> OrderBook: + snapshot: Dict[str, Any] = await self.get_order_book_data(trading_pair) + snapshot_timestamp: float = time.time() + snapshot_msg: OrderBookMessage = HitbtcOrderBook.snapshot_message_from_exchange( + snapshot, + snapshot_timestamp, + metadata={"trading_pair": trading_pair}) + order_book = self.order_book_create_function() + active_order_tracker: HitbtcActiveOrderTracker = HitbtcActiveOrderTracker() + bids, asks = active_order_tracker.convert_snapshot_message_to_order_book_row(snapshot_msg) + order_book.apply_snapshot(bids, asks, snapshot_msg.update_id) + return order_book + + async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for trades using websocket trade channel + """ + while True: + try: + ws = HitbtcWebsocket() + await ws.connect() + + for pair in self._trading_pairs: + await ws.subscribe(Constants.WS_SUB["TRADES"], convert_to_exchange_trading_pair(pair)) + + async for response in ws.on_message(): + method: str = response.get("method", None) + trades_data: str = response.get("params", None) + + if trades_data is None or method != Constants.WS_METHODS['TRADES_UPDATE']: + continue + + pair: str = convert_from_exchange_trading_pair(response["params"]["symbol"]) + + for trade in trades_data["data"]: + trade: Dict[Any] = trade + trade_timestamp: int = str_date_to_ts(trade["timestamp"]) + trade_msg: OrderBookMessage = HitbtcOrderBook.trade_message_from_exchange( + trade, + trade_timestamp, + metadata={"trading_pair": pair}) + output.put_nowait(trade_msg) + + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error.", exc_info=True) + await asyncio.sleep(5.0) + finally: + await ws.disconnect() + + async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for orderbook diffs using websocket book channel + """ + while True: + try: + ws = HitbtcWebsocket() + await ws.connect() + + order_book_methods = [ + Constants.WS_METHODS['ORDERS_SNAPSHOT'], + Constants.WS_METHODS['ORDERS_UPDATE'], + ] + + for pair in self._trading_pairs: + await ws.subscribe(Constants.WS_SUB["ORDERS"], convert_to_exchange_trading_pair(pair)) + + async for response in ws.on_message(): + method: str = response.get("method", None) + order_book_data: str = response.get("params", None) + + if order_book_data is None or method not in order_book_methods: + continue + + timestamp: int = str_date_to_ts(order_book_data["timestamp"]) + pair: str = convert_from_exchange_trading_pair(order_book_data["symbol"]) + + order_book_msg_cls = (HitbtcOrderBook.diff_message_from_exchange + if method == Constants.WS_METHODS['ORDERS_UPDATE'] else + HitbtcOrderBook.snapshot_message_from_exchange) + + orderbook_msg: OrderBookMessage = order_book_msg_cls( + order_book_data, + timestamp, + metadata={"trading_pair": pair}) + output.put_nowait(orderbook_msg) + + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unexpected error with WebSocket connection.", exc_info=True, + app_warning_msg="Unexpected error with WebSocket connection. Retrying in 30 seconds. " + "Check network connection.") + await asyncio.sleep(30.0) + finally: + await ws.disconnect() + + async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for orderbook snapshots by fetching orderbook + """ + while True: + try: + for trading_pair in self._trading_pairs: + try: + snapshot: Dict[str, any] = await self.get_order_book_data(trading_pair) + snapshot_timestamp: int = str_date_to_ts(snapshot["timestamp"]) + snapshot_msg: OrderBookMessage = HitbtcOrderBook.snapshot_message_from_exchange( + snapshot, + snapshot_timestamp, + metadata={"trading_pair": trading_pair} + ) + output.put_nowait(snapshot_msg) + self.logger().debug(f"Saved order book snapshot for {trading_pair}") + # Be careful not to go above API rate limits. + await asyncio.sleep(5.0) + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unexpected error with WebSocket connection.", exc_info=True, + app_warning_msg="Unexpected error with WebSocket connection. Retrying in 5 seconds. " + "Check network connection.") + await asyncio.sleep(5.0) + this_hour: pd.Timestamp = pd.Timestamp.utcnow().replace(minute=0, second=0, microsecond=0) + next_hour: pd.Timestamp = this_hour + pd.Timedelta(hours=1) + delta: float = next_hour.timestamp() - time.time() + await asyncio.sleep(delta) + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error.", exc_info=True) + await asyncio.sleep(5.0) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py b/hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py new file mode 100755 index 0000000000..954ab9c344 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +import time +import asyncio +import logging +from typing import ( + Any, + AsyncIterable, + List, + Optional, +) +from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource +from hummingbot.logger import HummingbotLogger +from .hitbtc_constants import Constants +from .hitbtc_auth import HitbtcAuth +from .hitbtc_utils import HitbtcAPIError +from .hitbtc_websocket import HitbtcWebsocket + + +class HitbtcAPIUserStreamDataSource(UserStreamTrackerDataSource): + + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, hitbtc_auth: HitbtcAuth, trading_pairs: Optional[List[str]] = []): + self._hitbtc_auth: HitbtcAuth = hitbtc_auth + self._ws: HitbtcWebsocket = None + self._trading_pairs = trading_pairs + self._current_listen_key = None + self._listen_for_user_stream_task = None + self._last_recv_time: float = 0 + super().__init__() + + @property + def last_recv_time(self) -> float: + return self._last_recv_time + + async def _ws_request_balances(self): + return await self._ws.request(Constants.WS_METHODS["USER_BALANCE"]) + + async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: + """ + Subscribe to active orders via web socket + """ + + try: + self._ws = HitbtcWebsocket(self._hitbtc_auth) + + await self._ws.connect() + + await self._ws.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"], None, {}) + + event_methods = [ + Constants.WS_METHODS["USER_ORDERS"], + Constants.WS_METHODS["USER_TRADES"], + ] + + async for msg in self._ws.on_message(): + self._last_recv_time = time.time() + + if msg.get("params", msg.get("result", None)) is None: + continue + elif msg.get("method", None) in event_methods: + await self._ws_request_balances() + yield msg + except Exception as e: + raise e + finally: + await self._ws.disconnect() + await asyncio.sleep(5) + + async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue) -> AsyncIterable[Any]: + """ + *required + Subscribe to user stream via web socket, and keep the connection open for incoming messages + :param ev_loop: ev_loop to execute this function in + :param output: an async queue where the incoming messages are stored + """ + + while True: + try: + async for msg in self._listen_to_orders_trades_balances(): + output.put_nowait(msg) + except asyncio.CancelledError: + raise + except HitbtcAPIError as e: + self.logger().error(e.error_payload.get('error'), exc_info=True) + raise + except Exception: + self.logger().error( + f"Unexpected error with {Constants.EXCHANGE_NAME} WebSocket connection. " + "Retrying after 30 seconds...", exc_info=True) + await asyncio.sleep(30.0) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_auth.py b/hummingbot/connector/exchange/coinzoom/coinzoom_auth.py new file mode 100755 index 0000000000..be37f2e149 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_auth.py @@ -0,0 +1,72 @@ +import hmac +import hashlib +import time +from base64 import b64encode +from typing import Dict, Any + + +class HitbtcAuth(): + """ + Auth class required by HitBTC API + Learn more at https://exchange-docs.crypto.com/#digital-signature + """ + def __init__(self, api_key: str, secret_key: str): + self.api_key = api_key + self.secret_key = secret_key + + def generate_payload( + self, + method: str, + url: str, + params: Dict[str, Any] = None, + ): + """ + Generates authentication payload and returns it. + :return: A base64 encoded payload for the authentication header. + """ + # Nonce is standard EPOCH timestamp only accurate to 1s + nonce = str(int(time.time())) + body = "" + # Need to build the full URL with query string for HS256 sig + if params is not None and len(params) > 0: + query_string = "&".join([f"{k}={v}" for k, v in params.items()]) + if method == "GET": + url = f"{url}?{query_string}" + else: + body = query_string + # Concat payload + payload = f"{method}{nonce}{url}{body}" + # Create HS256 sig + sig = hmac.new(self.secret_key.encode(), payload.encode(), hashlib.sha256).hexdigest() + # Base64 encode it with public key and nonce + return b64encode(f"{self.api_key}:{nonce}:{sig}".encode()).decode().strip() + + def generate_auth_dict_ws(self, + nonce: int): + """ + Generates an authentication params for HitBTC websockets login + :return: a dictionary of auth params + """ + return { + "algo": "HS256", + "pKey": str(self.api_key), + "nonce": str(nonce), + "signature": hmac.new(self.secret_key.encode('utf-8'), + str(nonce).encode('utf-8'), + hashlib.sha256).hexdigest() + } + + def get_headers(self, + method, + url, + params) -> Dict[str, Any]: + """ + Generates authentication headers required by HitBTC + :return: a dictionary of auth headers + """ + payload = self.generate_payload(method, url, params) + headers = { + "Authorization": f"HS256 {payload}", + "Content-Type": "application/x-www-form-urlencoded", + } + return headers diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py new file mode 100644 index 0000000000..538e0b21f2 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py @@ -0,0 +1,57 @@ +# A single source of truth for constant variables related to the exchange +class Constants: + EXCHANGE_NAME = "hitbtc" + REST_URL = "https://api.hitbtc.com/api/2" + REST_URL_AUTH = "/api/2" + WS_PRIVATE_URL = "wss://api.hitbtc.com/api/2/ws/trading" + WS_PUBLIC_URL = "wss://api.hitbtc.com/api/2/ws/public" + + HBOT_BROKER_ID = "refzzz48" + + ENDPOINT = { + # Public Endpoints + "TICKER": "public/ticker", + "TICKER_SINGLE": "public/ticker/{trading_pair}", + "SYMBOL": "public/symbol", + "ORDER_BOOK": "public/orderbook", + "ORDER_CREATE": "order", + "ORDER_DELETE": "order/{id}", + "ORDER_STATUS": "order/{id}", + "USER_ORDERS": "order", + "USER_BALANCES": "trading/balance", + } + + WS_SUB = { + "TRADES": "Trades", + "ORDERS": "Orderbook", + "USER_ORDERS_TRADES": "Reports", + + } + + WS_METHODS = { + "ORDERS_SNAPSHOT": "snapshotOrderbook", + "ORDERS_UPDATE": "updateOrderbook", + "TRADES_SNAPSHOT": "snapshotTrades", + "TRADES_UPDATE": "updateTrades", + "USER_BALANCE": "getTradingBalance", + "USER_ORDERS": "activeOrders", + "USER_TRADES": "report", + } + + # Timeouts + MESSAGE_TIMEOUT = 30.0 + PING_TIMEOUT = 10.0 + API_CALL_TIMEOUT = 10.0 + API_MAX_RETRIES = 4 + + # Intervals + # Only used when nothing is received from WS + SHORT_POLL_INTERVAL = 5.0 + # One minute should be fine since we get trades, orders and balances via WS + LONG_POLL_INTERVAL = 60.0 + UPDATE_ORDER_STATUS_INTERVAL = 60.0 + # 10 minute interval to update trading rules, these would likely never change whilst running. + INTERVAL_TRADING_RULES = 600 + + # Trading pair splitter regex + TRADING_PAIR_SPLITTER = r"^(\w+)(BTC|BCH|DAI|DDRST|EOSDT|EOS|ETH|EURS|IDRT|PAX|BUSD|GUSD|TUSD|USDC|USDT|USD)$" diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py new file mode 100644 index 0000000000..9f6f83ec15 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py @@ -0,0 +1,877 @@ +import logging +from typing import ( + Dict, + List, + Optional, + Any, + AsyncIterable, +) +from decimal import Decimal +import asyncio +import aiohttp +import math +import time +from async_timeout import timeout + +from hummingbot.core.network_iterator import NetworkStatus +from hummingbot.logger import HummingbotLogger +from hummingbot.core.clock import Clock +from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather +from hummingbot.connector.trading_rule import TradingRule +from hummingbot.core.data_type.cancellation_result import CancellationResult +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.limit_order import LimitOrder +from hummingbot.core.event.events import ( + MarketEvent, + BuyOrderCompletedEvent, + SellOrderCompletedEvent, + OrderFilledEvent, + OrderCancelledEvent, + BuyOrderCreatedEvent, + SellOrderCreatedEvent, + MarketOrderFailureEvent, + OrderType, + TradeType, + TradeFee +) +from hummingbot.connector.exchange_base import ExchangeBase +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_tracker import HitbtcOrderBookTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_user_stream_tracker import HitbtcUserStreamTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth +from hummingbot.connector.exchange.hitbtc.hitbtc_in_flight_order import HitbtcInFlightOrder +from hummingbot.connector.exchange.hitbtc.hitbtc_utils import ( + convert_from_exchange_trading_pair, + convert_to_exchange_trading_pair, + get_new_client_order_id, + aiohttp_response_with_errors, + retry_sleep_time, + str_date_to_ts, + HitbtcAPIError, +) +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants +from hummingbot.core.data_type.common import OpenOrder +ctce_logger = None +s_decimal_NaN = Decimal("nan") + + +class HitbtcExchange(ExchangeBase): + """ + HitbtcExchange connects with HitBTC exchange and provides order book pricing, user account tracking and + trading functionality. + """ + ORDER_NOT_EXIST_CONFIRMATION_COUNT = 3 + ORDER_NOT_EXIST_CANCEL_COUNT = 2 + + @classmethod + def logger(cls) -> HummingbotLogger: + global ctce_logger + if ctce_logger is None: + ctce_logger = logging.getLogger(__name__) + return ctce_logger + + def __init__(self, + hitbtc_api_key: str, + hitbtc_secret_key: str, + trading_pairs: Optional[List[str]] = None, + trading_required: bool = True + ): + """ + :param hitbtc_api_key: The API key to connect to private HitBTC APIs. + :param hitbtc_secret_key: The API secret. + :param trading_pairs: The market trading pairs which to track order book data. + :param trading_required: Whether actual trading is needed. + """ + super().__init__() + self._trading_required = trading_required + self._trading_pairs = trading_pairs + self._hitbtc_auth = HitbtcAuth(hitbtc_api_key, hitbtc_secret_key) + self._order_book_tracker = HitbtcOrderBookTracker(trading_pairs=trading_pairs) + self._user_stream_tracker = HitbtcUserStreamTracker(self._hitbtc_auth, trading_pairs) + self._ev_loop = asyncio.get_event_loop() + self._shared_client = None + self._poll_notifier = asyncio.Event() + self._last_timestamp = 0 + self._in_flight_orders = {} # Dict[client_order_id:str, HitbtcInFlightOrder] + self._order_not_found_records = {} # Dict[client_order_id:str, count:int] + self._trading_rules = {} # Dict[trading_pair:str, TradingRule] + self._status_polling_task = None + self._user_stream_event_listener_task = None + self._trading_rules_polling_task = None + self._last_poll_timestamp = 0 + + @property + def name(self) -> str: + return "hitbtc" + + @property + def order_books(self) -> Dict[str, OrderBook]: + return self._order_book_tracker.order_books + + @property + def trading_rules(self) -> Dict[str, TradingRule]: + return self._trading_rules + + @property + def in_flight_orders(self) -> Dict[str, HitbtcInFlightOrder]: + return self._in_flight_orders + + @property + def status_dict(self) -> Dict[str, bool]: + """ + A dictionary of statuses of various connector's components. + """ + return { + "order_books_initialized": self._order_book_tracker.ready, + "account_balance": len(self._account_balances) > 0 if self._trading_required else True, + "trading_rule_initialized": len(self._trading_rules) > 0, + "user_stream_initialized": + self._user_stream_tracker.data_source.last_recv_time > 0 if self._trading_required else True, + } + + @property + def ready(self) -> bool: + """ + :return True when all statuses pass, this might take 5-10 seconds for all the connector's components and + services to be ready. + """ + return all(self.status_dict.values()) + + @property + def limit_orders(self) -> List[LimitOrder]: + return [ + in_flight_order.to_limit_order() + for in_flight_order in self._in_flight_orders.values() + ] + + @property + def tracking_states(self) -> Dict[str, any]: + """ + :return active in-flight orders in json format, is used to save in sqlite db. + """ + return { + key: value.to_json() + for key, value in self._in_flight_orders.items() + if not value.is_done + } + + def restore_tracking_states(self, saved_states: Dict[str, any]): + """ + Restore in-flight orders from saved tracking states, this is st the connector can pick up on where it left off + when it disconnects. + :param saved_states: The saved tracking_states. + """ + self._in_flight_orders.update({ + key: HitbtcInFlightOrder.from_json(value) + for key, value in saved_states.items() + }) + + def supported_order_types(self) -> List[OrderType]: + """ + :return a list of OrderType supported by this connector. + Note that Market order type is no longer required and will not be used. + """ + return [OrderType.LIMIT, OrderType.LIMIT_MAKER] + + def start(self, clock: Clock, timestamp: float): + """ + This function is called automatically by the clock. + """ + super().start(clock, timestamp) + + def stop(self, clock: Clock): + """ + This function is called automatically by the clock. + """ + super().stop(clock) + + async def start_network(self): + """ + This function is required by NetworkIterator base class and is called automatically. + It starts tracking order book, polling trading rules, + updating statuses and tracking user data. + """ + self._order_book_tracker.start() + self._trading_rules_polling_task = safe_ensure_future(self._trading_rules_polling_loop()) + if self._trading_required: + self._status_polling_task = safe_ensure_future(self._status_polling_loop()) + self._user_stream_tracker_task = safe_ensure_future(self._user_stream_tracker.start()) + self._user_stream_event_listener_task = safe_ensure_future(self._user_stream_event_listener()) + + async def stop_network(self): + """ + This function is required by NetworkIterator base class and is called automatically. + """ + self._order_book_tracker.stop() + if self._status_polling_task is not None: + self._status_polling_task.cancel() + self._status_polling_task = None + if self._trading_rules_polling_task is not None: + self._trading_rules_polling_task.cancel() + self._trading_rules_polling_task = None + if self._status_polling_task is not None: + self._status_polling_task.cancel() + self._status_polling_task = None + if self._user_stream_tracker_task is not None: + self._user_stream_tracker_task.cancel() + self._user_stream_tracker_task = None + if self._user_stream_event_listener_task is not None: + self._user_stream_event_listener_task.cancel() + self._user_stream_event_listener_task = None + + async def check_network(self) -> NetworkStatus: + """ + This function is required by NetworkIterator base class and is called periodically to check + the network connection. Simply ping the network (or call any light weight public API). + """ + try: + # since there is no ping endpoint, the lowest rate call is to get BTC-USD symbol + await self._api_request("GET", + Constants.ENDPOINT['SYMBOL'], + params={'symbols': 'BTCUSD'}) + except asyncio.CancelledError: + raise + except Exception: + return NetworkStatus.NOT_CONNECTED + return NetworkStatus.CONNECTED + + async def _http_client(self) -> aiohttp.ClientSession: + """ + :returns Shared client session instance + """ + if self._shared_client is None: + self._shared_client = aiohttp.ClientSession() + return self._shared_client + + async def _trading_rules_polling_loop(self): + """ + Periodically update trading rule. + """ + while True: + try: + await self._update_trading_rules() + await asyncio.sleep(Constants.INTERVAL_TRADING_RULES) + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().network(f"Unexpected error while fetching trading rules. Error: {str(e)}", + exc_info=True, + app_warning_msg=("Could not fetch new trading rules from " + f"{Constants.EXCHANGE_NAME}. Check network connection.")) + await asyncio.sleep(0.5) + + async def _update_trading_rules(self): + symbols_info = await self._api_request("GET", endpoint=Constants.ENDPOINT['SYMBOL']) + self._trading_rules.clear() + self._trading_rules = self._format_trading_rules(symbols_info) + + def _format_trading_rules(self, symbols_info: Dict[str, Any]) -> Dict[str, TradingRule]: + """ + Converts json API response into a dictionary of trading rules. + :param symbols_info: The json API response + :return A dictionary of trading rules. + Response Example: + [ + { + id: "BTCUSD", + baseCurrency: "BTC", + quoteCurrency: "USD", + quantityIncrement: "0.00001", + tickSize: "0.01", + takeLiquidityRate: "0.0025", + provideLiquidityRate: "0.001", + feeCurrency: "USD", + marginTrading: true, + maxInitialLeverage: "12.00" + } + ] + """ + result = {} + for rule in symbols_info: + try: + trading_pair = convert_from_exchange_trading_pair(rule["id"]) + price_step = Decimal(str(rule["tickSize"])) + size_step = Decimal(str(rule["quantityIncrement"])) + result[trading_pair] = TradingRule(trading_pair, + min_order_size=size_step, + min_base_amount_increment=size_step, + min_price_increment=price_step) + except Exception: + self.logger().error(f"Error parsing the trading pair rule {rule}. Skipping.", exc_info=True) + return result + + async def _api_request(self, + method: str, + endpoint: str, + params: Optional[Dict[str, Any]] = None, + is_auth_required: bool = False, + try_count: int = 0) -> Dict[str, Any]: + """ + Sends an aiohttp request and waits for a response. + :param method: The HTTP method, e.g. get or post + :param endpoint: The path url or the API end point + :param params: Additional get/post parameters + :param is_auth_required: Whether an authentication is required, when True the function will add encrypted + signature to the request. + :returns A response in json format. + """ + url = f"{Constants.REST_URL}/{endpoint}" + shared_client = await self._http_client() + # Turn `params` into either GET params or POST body data + qs_params: dict = params if method.upper() == "GET" else None + req_form = aiohttp.FormData(params) if method.upper() == "POST" and params is not None else None + # Generate auth headers if needed. + headers: dict = {"Content-Type": "application/x-www-form-urlencoded"} + if is_auth_required: + headers: dict = self._hitbtc_auth.get_headers(method, f"{Constants.REST_URL_AUTH}/{endpoint}", + params) + # Build request coro + response_coro = shared_client.request(method=method.upper(), url=url, headers=headers, + params=qs_params, data=req_form, + timeout=Constants.API_CALL_TIMEOUT) + http_status, parsed_response, request_errors = await aiohttp_response_with_errors(response_coro) + if request_errors or parsed_response is None: + if try_count < Constants.API_MAX_RETRIES: + try_count += 1 + time_sleep = retry_sleep_time(try_count) + self.logger().info(f"Error fetching data from {url}. HTTP status is {http_status}. " + f"Retrying in {time_sleep:.0f}s.") + await asyncio.sleep(time_sleep) + return await self._api_request(method=method, endpoint=endpoint, params=params, + is_auth_required=is_auth_required, try_count=try_count) + else: + raise HitbtcAPIError({"error": parsed_response, "status": http_status}) + if "error" in parsed_response: + raise HitbtcAPIError(parsed_response) + return parsed_response + + def get_order_price_quantum(self, trading_pair: str, price: Decimal): + """ + Returns a price step, a minimum price increment for a given trading pair. + """ + trading_rule = self._trading_rules[trading_pair] + return trading_rule.min_price_increment + + def get_order_size_quantum(self, trading_pair: str, order_size: Decimal): + """ + Returns an order amount step, a minimum amount increment for a given trading pair. + """ + trading_rule = self._trading_rules[trading_pair] + return Decimal(trading_rule.min_base_amount_increment) + + def get_order_book(self, trading_pair: str) -> OrderBook: + if trading_pair not in self._order_book_tracker.order_books: + raise ValueError(f"No order book exists for '{trading_pair}'.") + return self._order_book_tracker.order_books[trading_pair] + + def buy(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, + price: Decimal = s_decimal_NaN, **kwargs) -> str: + """ + Buys an amount of base asset (of the given trading pair). This function returns immediately. + To see an actual order, you'll have to wait for BuyOrderCreatedEvent. + :param trading_pair: The market (e.g. BTC-USDT) to buy from + :param amount: The amount in base token value + :param order_type: The order type + :param price: The price (note: this is no longer optional) + :returns A new internal order id + """ + order_id: str = get_new_client_order_id(True, trading_pair) + safe_ensure_future(self._create_order(TradeType.BUY, order_id, trading_pair, amount, order_type, price)) + return order_id + + def sell(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, + price: Decimal = s_decimal_NaN, **kwargs) -> str: + """ + Sells an amount of base asset (of the given trading pair). This function returns immediately. + To see an actual order, you'll have to wait for SellOrderCreatedEvent. + :param trading_pair: The market (e.g. BTC-USDT) to sell from + :param amount: The amount in base token value + :param order_type: The order type + :param price: The price (note: this is no longer optional) + :returns A new internal order id + """ + order_id: str = get_new_client_order_id(False, trading_pair) + safe_ensure_future(self._create_order(TradeType.SELL, order_id, trading_pair, amount, order_type, price)) + return order_id + + def cancel(self, trading_pair: str, order_id: str): + """ + Cancel an order. This function returns immediately. + To get the cancellation result, you'll have to wait for OrderCancelledEvent. + :param trading_pair: The market (e.g. BTC-USDT) of the order. + :param order_id: The internal order id (also called client_order_id) + """ + safe_ensure_future(self._execute_cancel(trading_pair, order_id)) + return order_id + + async def _create_order(self, + trade_type: TradeType, + order_id: str, + trading_pair: str, + amount: Decimal, + order_type: OrderType, + price: Decimal): + """ + Calls create-order API end point to place an order, starts tracking the order and triggers order created event. + :param trade_type: BUY or SELL + :param order_id: Internal order id (also called client_order_id) + :param trading_pair: The market to place order + :param amount: The order amount (in base token value) + :param order_type: The order type + :param price: The order price + """ + if not order_type.is_limit_type(): + raise Exception(f"Unsupported order type: {order_type}") + trading_rule = self._trading_rules[trading_pair] + + amount = self.quantize_order_amount(trading_pair, amount) + price = self.quantize_order_price(trading_pair, price) + if amount < trading_rule.min_order_size: + raise ValueError(f"Buy order amount {amount} is lower than the minimum order size " + f"{trading_rule.min_order_size}.") + order_type_str = order_type.name.lower().split("_")[0] + api_params = {"symbol": convert_to_exchange_trading_pair(trading_pair), + "side": trade_type.name.lower(), + "type": order_type_str, + "price": f"{price:f}", + "quantity": f"{amount:f}", + "clientOrderId": order_id, + # Without strict validate, HitBTC might adjust order prices/sizes. + "strictValidate": "true", + } + if order_type is OrderType.LIMIT_MAKER: + api_params["postOnly"] = "true" + self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, order_type) + try: + order_result = await self._api_request("POST", Constants.ENDPOINT["ORDER_CREATE"], api_params, True) + exchange_order_id = str(order_result["id"]) + tracked_order = self._in_flight_orders.get(order_id) + if tracked_order is not None: + self.logger().info(f"Created {order_type.name} {trade_type.name} order {order_id} for " + f"{amount} {trading_pair}.") + tracked_order.update_exchange_order_id(exchange_order_id) + if trade_type is TradeType.BUY: + event_tag = MarketEvent.BuyOrderCreated + event_cls = BuyOrderCreatedEvent + else: + event_tag = MarketEvent.SellOrderCreated + event_cls = SellOrderCreatedEvent + self.trigger_event(event_tag, + event_cls(self.current_timestamp, order_type, trading_pair, amount, price, order_id)) + except asyncio.CancelledError: + raise + except HitbtcAPIError as e: + error_reason = e.error_payload.get('error', {}).get('message') + self.stop_tracking_order(order_id) + self.logger().network( + f"Error submitting {trade_type.name} {order_type.name} order to {Constants.EXCHANGE_NAME} for " + f"{amount} {trading_pair} {price} - {error_reason}.", + exc_info=True, + app_warning_msg=(f"Error submitting order to {Constants.EXCHANGE_NAME} - {error_reason}.") + ) + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent(self.current_timestamp, order_id, order_type)) + + def start_tracking_order(self, + order_id: str, + exchange_order_id: str, + trading_pair: str, + trade_type: TradeType, + price: Decimal, + amount: Decimal, + order_type: OrderType): + """ + Starts tracking an order by simply adding it into _in_flight_orders dictionary. + """ + self._in_flight_orders[order_id] = HitbtcInFlightOrder( + client_order_id=order_id, + exchange_order_id=exchange_order_id, + trading_pair=trading_pair, + order_type=order_type, + trade_type=trade_type, + price=price, + amount=amount + ) + + def stop_tracking_order(self, order_id: str): + """ + Stops tracking an order by simply removing it from _in_flight_orders dictionary. + """ + if order_id in self._in_flight_orders: + del self._in_flight_orders[order_id] + if order_id in self._order_not_found_records: + del self._order_not_found_records[order_id] + + async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: + """ + Executes order cancellation process by first calling cancel-order API. The API result doesn't confirm whether + the cancellation is successful, it simply states it receives the request. + :param trading_pair: The market trading pair (Unused during cancel on HitBTC) + :param order_id: The internal order id + order.last_state to change to CANCELED + """ + order_was_cancelled = False + try: + tracked_order = self._in_flight_orders.get(order_id) + if tracked_order is None: + raise ValueError(f"Failed to cancel order - {order_id}. Order not found.") + if tracked_order.exchange_order_id is None: + await tracked_order.get_exchange_order_id() + # ex_order_id = tracked_order.exchange_order_id + await self._api_request("DELETE", + Constants.ENDPOINT["ORDER_DELETE"].format(id=order_id), + is_auth_required=True) + order_was_cancelled = True + except asyncio.CancelledError: + raise + except HitbtcAPIError as e: + err = e.error_payload.get('error', e.error_payload) + self._order_not_found_records[order_id] = self._order_not_found_records.get(order_id, 0) + 1 + if err.get('code') == 20002 and \ + self._order_not_found_records[order_id] >= self.ORDER_NOT_EXIST_CANCEL_COUNT: + order_was_cancelled = True + if order_was_cancelled: + self.logger().info(f"Successfully cancelled order {order_id} on {Constants.EXCHANGE_NAME}.") + self.stop_tracking_order(order_id) + self.trigger_event(MarketEvent.OrderCancelled, + OrderCancelledEvent(self.current_timestamp, order_id)) + tracked_order.cancelled_event.set() + return CancellationResult(order_id, True) + else: + self.logger().network( + f"Failed to cancel order {order_id}: {err.get('message', str(err))}", + exc_info=True, + app_warning_msg=f"Failed to cancel the order {order_id} on {Constants.EXCHANGE_NAME}. " + f"Check API key and network connection." + ) + return CancellationResult(order_id, False) + + async def _status_polling_loop(self): + """ + Periodically update user balances and order status via REST API. This serves as a fallback measure for web + socket API updates. + """ + while True: + try: + self._poll_notifier = asyncio.Event() + await self._poll_notifier.wait() + await safe_gather( + self._update_balances(), + self._update_order_status(), + ) + self._last_poll_timestamp = self.current_timestamp + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().error(str(e), exc_info=True) + warn_msg = (f"Could not fetch account updates from {Constants.EXCHANGE_NAME}. " + "Check API key and network connection.") + self.logger().network("Unexpected error while fetching account updates.", exc_info=True, + app_warning_msg=warn_msg) + await asyncio.sleep(0.5) + + async def _update_balances(self): + """ + Calls REST API to update total and available balances. + """ + account_info = await self._api_request("GET", Constants.ENDPOINT["USER_BALANCES"], is_auth_required=True) + self._process_balance_message(account_info) + + async def _update_order_status(self): + """ + Calls REST API to get status update for each in-flight order. + """ + last_tick = int(self._last_poll_timestamp / Constants.UPDATE_ORDER_STATUS_INTERVAL) + current_tick = int(self.current_timestamp / Constants.UPDATE_ORDER_STATUS_INTERVAL) + + if current_tick > last_tick and len(self._in_flight_orders) > 0: + tracked_orders = list(self._in_flight_orders.values()) + tasks = [] + for tracked_order in tracked_orders: + # exchange_order_id = await tracked_order.get_exchange_order_id() + order_id = tracked_order.client_order_id + tasks.append(self._api_request("GET", + Constants.ENDPOINT["ORDER_STATUS"].format(id=order_id), + is_auth_required=True)) + self.logger().debug(f"Polling for order status updates of {len(tasks)} orders.") + responses = await safe_gather(*tasks, return_exceptions=True) + for response, tracked_order in zip(responses, tracked_orders): + client_order_id = tracked_order.client_order_id + if isinstance(response, HitbtcAPIError): + err = response.error_payload.get('error', response.error_payload) + if err.get('code') == 20002: + self._order_not_found_records[client_order_id] = \ + self._order_not_found_records.get(client_order_id, 0) + 1 + if self._order_not_found_records[client_order_id] < self.ORDER_NOT_EXIST_CONFIRMATION_COUNT: + # Wait until the order not found error have repeated a few times before actually treating + # it as failed. See: https://github.com/CoinAlpha/hummingbot/issues/601 + continue + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent( + self.current_timestamp, client_order_id, tracked_order.order_type)) + self.stop_tracking_order(client_order_id) + else: + continue + elif "clientOrderId" not in response: + self.logger().info(f"_update_order_status clientOrderId not in resp: {response}") + continue + else: + self._process_order_message(response) + + def _process_order_message(self, order_msg: Dict[str, Any]): + """ + Updates in-flight order and triggers cancellation or failure event if needed. + :param order_msg: The order response from either REST or web socket API (they are of the same format) + Example Order: + { + "id": "4345613661", + "clientOrderId": "57d5525562c945448e3cbd559bd068c3", + "symbol": "BCCBTC", + "side": "sell", + "status": "new", + "type": "limit", + "timeInForce": "GTC", + "quantity": "0.013", + "price": "0.100000", + "cumQuantity": "0.000", + "postOnly": false, + "createdAt": "2017-10-20T12:17:12.245Z", + "updatedAt": "2017-10-20T12:17:12.245Z", + "reportType": "status" + } + """ + client_order_id = order_msg["clientOrderId"] + if client_order_id not in self._in_flight_orders: + return + tracked_order = self._in_flight_orders[client_order_id] + # Update order execution status + tracked_order.last_state = order_msg["status"] + # update order + tracked_order.executed_amount_base = Decimal(order_msg["cumQuantity"]) + tracked_order.executed_amount_quote = Decimal(order_msg["price"]) * Decimal(order_msg["cumQuantity"]) + + if tracked_order.is_cancelled: + self.logger().info(f"Successfully cancelled order {client_order_id}.") + self.stop_tracking_order(client_order_id) + self.trigger_event(MarketEvent.OrderCancelled, + OrderCancelledEvent(self.current_timestamp, client_order_id)) + tracked_order.cancelled_event.set() + elif tracked_order.is_failure: + self.logger().info(f"The market order {client_order_id} has failed according to order status API. ") + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent( + self.current_timestamp, client_order_id, tracked_order.order_type)) + self.stop_tracking_order(client_order_id) + + async def _process_trade_message(self, trade_msg: Dict[str, Any]): + """ + Updates in-flight order and trigger order filled event for trade message received. Triggers order completed + event if the total executed amount equals to the specified order amount. + Example Trade: + { + "id": "4345697765", + "clientOrderId": "53b7cf917963464a811a4af426102c19", + "symbol": "ETHBTC", + "side": "sell", + "status": "filled", + "type": "limit", + "timeInForce": "GTC", + "quantity": "0.001", + "price": "0.053868", + "cumQuantity": "0.001", + "postOnly": false, + "createdAt": "2017-10-20T12:20:05.952Z", + "updatedAt": "2017-10-20T12:20:38.708Z", + "reportType": "trade", + "tradeQuantity": "0.001", + "tradePrice": "0.053868", + "tradeId": 55051694, + "tradeFee": "-0.000000005" + } + """ + tracked_orders = list(self._in_flight_orders.values()) + for order in tracked_orders: + await order.get_exchange_order_id() + track_order = [o for o in tracked_orders if trade_msg["id"] == o.exchange_order_id] + if not track_order: + return + tracked_order = track_order[0] + updated = tracked_order.update_with_trade_update(trade_msg) + if not updated: + return + self.trigger_event( + MarketEvent.OrderFilled, + OrderFilledEvent( + self.current_timestamp, + tracked_order.client_order_id, + tracked_order.trading_pair, + tracked_order.trade_type, + tracked_order.order_type, + Decimal(str(trade_msg.get("tradePrice", "0"))), + Decimal(str(trade_msg.get("tradeQuantity", "0"))), + TradeFee(0.0, [(tracked_order.quote_asset, Decimal(str(trade_msg.get("tradeFee", "0"))))]), + exchange_trade_id=trade_msg["id"] + ) + ) + if math.isclose(tracked_order.executed_amount_base, tracked_order.amount) or \ + tracked_order.executed_amount_base >= tracked_order.amount or \ + tracked_order.is_done: + tracked_order.last_state = "FILLED" + self.logger().info(f"The {tracked_order.trade_type.name} order " + f"{tracked_order.client_order_id} has completed " + f"according to order status API.") + event_tag = MarketEvent.BuyOrderCompleted if tracked_order.trade_type is TradeType.BUY \ + else MarketEvent.SellOrderCompleted + event_class = BuyOrderCompletedEvent if tracked_order.trade_type is TradeType.BUY \ + else SellOrderCompletedEvent + await asyncio.sleep(0.1) + self.trigger_event(event_tag, + event_class(self.current_timestamp, + tracked_order.client_order_id, + tracked_order.base_asset, + tracked_order.quote_asset, + tracked_order.fee_asset, + tracked_order.executed_amount_base, + tracked_order.executed_amount_quote, + tracked_order.fee_paid, + tracked_order.order_type)) + self.stop_tracking_order(tracked_order.client_order_id) + + def _process_balance_message(self, balance_update): + local_asset_names = set(self._account_balances.keys()) + remote_asset_names = set() + for account in balance_update: + asset_name = account["currency"] + self._account_available_balances[asset_name] = Decimal(str(account["available"])) + self._account_balances[asset_name] = Decimal(str(account["reserved"])) + Decimal(str(account["available"])) + remote_asset_names.add(asset_name) + + asset_names_to_remove = local_asset_names.difference(remote_asset_names) + for asset_name in asset_names_to_remove: + del self._account_available_balances[asset_name] + del self._account_balances[asset_name] + + async def cancel_all(self, timeout_seconds: float) -> List[CancellationResult]: + """ + Cancels all in-flight orders and waits for cancellation results. + Used by bot's top level stop and exit commands (cancelling outstanding orders on exit) + :param timeout_seconds: The timeout at which the operation will be canceled. + :returns List of CancellationResult which indicates whether each order is successfully cancelled. + """ + if self._trading_pairs is None: + raise Exception("cancel_all can only be used when trading_pairs are specified.") + open_orders = [o for o in self._in_flight_orders.values() if not o.is_done] + if len(open_orders) == 0: + return [] + tasks = [self._execute_cancel(o.trading_pair, o.client_order_id) for o in open_orders] + cancellation_results = [] + try: + async with timeout(timeout_seconds): + cancellation_results = await safe_gather(*tasks, return_exceptions=False) + except Exception: + self.logger().network( + "Unexpected error cancelling orders.", exc_info=True, + app_warning_msg=(f"Failed to cancel all orders on {Constants.EXCHANGE_NAME}. " + "Check API key and network connection.") + ) + return cancellation_results + + def tick(self, timestamp: float): + """ + Is called automatically by the clock for each clock's tick (1 second by default). + It checks if status polling task is due for execution. + """ + now = time.time() + poll_interval = (Constants.SHORT_POLL_INTERVAL + if now - self._user_stream_tracker.last_recv_time > 60.0 + else Constants.LONG_POLL_INTERVAL) + last_tick = int(self._last_timestamp / poll_interval) + current_tick = int(timestamp / poll_interval) + if current_tick > last_tick: + if not self._poll_notifier.is_set(): + self._poll_notifier.set() + self._last_timestamp = timestamp + + def get_fee(self, + base_currency: str, + quote_currency: str, + order_type: OrderType, + order_side: TradeType, + amount: Decimal, + price: Decimal = s_decimal_NaN) -> TradeFee: + """ + To get trading fee, this function is simplified by using fee override configuration. Most parameters to this + function are ignore except order_type. Use OrderType.LIMIT_MAKER to specify you want trading fee for + maker order. + """ + is_maker = order_type is OrderType.LIMIT_MAKER + return TradeFee(percent=self.estimate_fee_pct(is_maker)) + + async def _iter_user_event_queue(self) -> AsyncIterable[Dict[str, any]]: + while True: + try: + yield await self._user_stream_tracker.user_stream.get() + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unknown error. Retrying after 1 seconds.", exc_info=True, + app_warning_msg=(f"Could not fetch user events from {Constants.EXCHANGE_NAME}. " + "Check API key and network connection.")) + await asyncio.sleep(1.0) + + async def _user_stream_event_listener(self): + """ + Listens to message in _user_stream_tracker.user_stream queue. The messages are put in by + HitbtcAPIUserStreamDataSource. + """ + async for event_message in self._iter_user_event_queue(): + try: + event_methods = [ + Constants.WS_METHODS["USER_ORDERS"], + Constants.WS_METHODS["USER_TRADES"], + ] + method: str = event_message.get("method", None) + params: str = event_message.get("params", None) + account_balances: list = event_message.get("result", None) + + if method not in event_methods and account_balances is None: + self.logger().error(f"Unexpected message in user stream: {event_message}.", exc_info=True) + continue + if method == Constants.WS_METHODS["USER_TRADES"]: + await self._process_trade_message(params) + elif method == Constants.WS_METHODS["USER_ORDERS"]: + for order_msg in params: + self._process_order_message(order_msg) + elif isinstance(account_balances, list) and "currency" in account_balances[0]: + self._process_balance_message(account_balances) + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error in user stream listener loop.", exc_info=True) + await asyncio.sleep(5.0) + + # This is currently unused, but looks like a future addition. + async def get_open_orders(self) -> List[OpenOrder]: + result = await self._api_request("GET", Constants.ENDPOINT["USER_ORDERS"], is_auth_required=True) + ret_val = [] + for order in result: + if Constants.HBOT_BROKER_ID not in order["clientOrderId"]: + continue + if order["type"] != OrderType.LIMIT.name.lower(): + self.logger().info(f"Unsupported order type found: {order['type']}") + continue + ret_val.append( + OpenOrder( + client_order_id=order["clientOrderId"], + trading_pair=convert_from_exchange_trading_pair(order["symbol"]), + price=Decimal(str(order["price"])), + amount=Decimal(str(order["quantity"])), + executed_amount=Decimal(str(order["cumQuantity"])), + status=order["status"], + order_type=OrderType.LIMIT, + is_buy=True if order["side"].lower() == TradeType.BUY.name.lower() else False, + time=str_date_to_ts(order["createdAt"]), + exchange_order_id=order["id"] + ) + ) + return ret_val diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py b/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py new file mode 100644 index 0000000000..54766be2f1 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py @@ -0,0 +1,118 @@ +from decimal import Decimal +from typing import ( + Any, + Dict, + Optional, +) +import asyncio +from hummingbot.core.event.events import ( + OrderType, + TradeType +) +from hummingbot.connector.in_flight_order_base import InFlightOrderBase + +s_decimal_0 = Decimal(0) + + +class HitbtcInFlightOrder(InFlightOrderBase): + def __init__(self, + client_order_id: str, + exchange_order_id: Optional[str], + trading_pair: str, + order_type: OrderType, + trade_type: TradeType, + price: Decimal, + amount: Decimal, + initial_state: str = "new"): + super().__init__( + client_order_id, + exchange_order_id, + trading_pair, + order_type, + trade_type, + price, + amount, + initial_state, + ) + self.trade_id_set = set() + self.cancelled_event = asyncio.Event() + + @property + def is_done(self) -> bool: + return self.last_state in {"filled", "canceled", "expired"} + + @property + def is_failure(self) -> bool: + return self.last_state in {"suspended"} + + @property + def is_cancelled(self) -> bool: + return self.last_state in {"canceled", "expired"} + + @classmethod + def from_json(cls, data: Dict[str, Any]) -> InFlightOrderBase: + """ + :param data: json data from API + :return: formatted InFlightOrder + """ + retval = HitbtcInFlightOrder( + data["client_order_id"], + data["exchange_order_id"], + data["trading_pair"], + getattr(OrderType, data["order_type"]), + getattr(TradeType, data["trade_type"]), + Decimal(data["price"]), + Decimal(data["amount"]), + data["last_state"] + ) + retval.executed_amount_base = Decimal(data["executed_amount_base"]) + retval.executed_amount_quote = Decimal(data["executed_amount_quote"]) + retval.fee_asset = data["fee_asset"] + retval.fee_paid = Decimal(data["fee_paid"]) + retval.last_state = data["last_state"] + return retval + + def update_with_trade_update(self, trade_update: Dict[str, Any]) -> bool: + """ + Updates the in flight order with trade update (from private/get-order-detail end point) + return: True if the order gets updated otherwise False + Example Trade: + { + "id": "4345697765", + "clientOrderId": "53b7cf917963464a811a4af426102c19", + "symbol": "ETHBTC", + "side": "sell", + "status": "filled", + "type": "limit", + "timeInForce": "GTC", + "quantity": "0.001", + "price": "0.053868", + "cumQuantity": "0.001", + "postOnly": false, + "createdAt": "2017-10-20T12:20:05.952Z", + "updatedAt": "2017-10-20T12:20:38.708Z", + "reportType": "trade", + } + ... Trade variables are only included after fills. + { + "tradeQuantity": "0.001", + "tradePrice": "0.053868", + "tradeId": 55051694, + "tradeFee": "-0.000000005" + } + """ + self.executed_amount_base = Decimal(str(trade_update["cumQuantity"])) + if self.executed_amount_base <= s_decimal_0: + # No trades executed yet. + return False + trade_id = trade_update["updatedAt"] + if trade_id in self.trade_id_set: + # trade already recorded + return False + self.trade_id_set.add(trade_id) + self.fee_paid += Decimal(str(trade_update.get("tradeFee", "0"))) + self.executed_amount_quote += (Decimal(str(trade_update.get("tradePrice", "0"))) * + Decimal(str(trade_update.get("tradeQuantity", "0")))) + if not self.fee_asset: + self.fee_asset = self.quote_asset + return True diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book.py b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book.py new file mode 100644 index 0000000000..1a3c91a121 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python + +import logging +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants + +from sqlalchemy.engine import RowProxy +from typing import ( + Optional, + Dict, + List, Any) +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_message import ( + OrderBookMessage, OrderBookMessageType +) +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_message import HitbtcOrderBookMessage + +_logger = None + + +class HitbtcOrderBook(OrderBook): + @classmethod + def logger(cls) -> HummingbotLogger: + global _logger + if _logger is None: + _logger = logging.getLogger(__name__) + return _logger + + @classmethod + def snapshot_message_from_exchange(cls, + msg: Dict[str, any], + timestamp: float, + metadata: Optional[Dict] = None): + """ + Convert json snapshot data into standard OrderBookMessage format + :param msg: json snapshot data from live web socket stream + :param timestamp: timestamp attached to incoming data + :return: HitbtcOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + return HitbtcOrderBookMessage( + message_type=OrderBookMessageType.SNAPSHOT, + content=msg, + timestamp=timestamp + ) + + @classmethod + def snapshot_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of snapshot data into standard OrderBookMessage format + :param record: a row of snapshot data from the database + :return: HitbtcOrderBookMessage + """ + return HitbtcOrderBookMessage( + message_type=OrderBookMessageType.SNAPSHOT, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def diff_message_from_exchange(cls, + msg: Dict[str, any], + timestamp: Optional[float] = None, + metadata: Optional[Dict] = None): + """ + Convert json diff data into standard OrderBookMessage format + :param msg: json diff data from live web socket stream + :param timestamp: timestamp attached to incoming data + :return: HitbtcOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + return HitbtcOrderBookMessage( + message_type=OrderBookMessageType.DIFF, + content=msg, + timestamp=timestamp + ) + + @classmethod + def diff_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of diff data into standard OrderBookMessage format + :param record: a row of diff data from the database + :return: HitbtcOrderBookMessage + """ + return HitbtcOrderBookMessage( + message_type=OrderBookMessageType.DIFF, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def trade_message_from_exchange(cls, + msg: Dict[str, Any], + timestamp: Optional[float] = None, + metadata: Optional[Dict] = None): + """ + Convert a trade data into standard OrderBookMessage format + :param record: a trade data from the database + :return: HitbtcOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + msg.update({ + "exchange_order_id": msg.get("id"), + "trade_type": msg.get("side"), + "price": msg.get("price"), + "amount": msg.get("quantity"), + }) + + return HitbtcOrderBookMessage( + message_type=OrderBookMessageType.TRADE, + content=msg, + timestamp=timestamp + ) + + @classmethod + def trade_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of trade data into standard OrderBookMessage format + :param record: a row of trade data from the database + :return: HitbtcOrderBookMessage + """ + return HitbtcOrderBookMessage( + message_type=OrderBookMessageType.TRADE, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def from_snapshot(cls, snapshot: OrderBookMessage): + raise NotImplementedError(Constants.EXCHANGE_NAME + " order book needs to retain individual order data.") + + @classmethod + def restore_from_snapshot_and_diffs(self, snapshot: OrderBookMessage, diffs: List[OrderBookMessage]): + raise NotImplementedError(Constants.EXCHANGE_NAME + " order book needs to retain individual order data.") diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py new file mode 100644 index 0000000000..1f0bc1d631 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python + +from typing import ( + Dict, + List, + Optional, +) + +from hummingbot.core.data_type.order_book_row import OrderBookRow +from hummingbot.core.data_type.order_book_message import ( + OrderBookMessage, + OrderBookMessageType, +) +from .hitbtc_utils import ( + convert_from_exchange_trading_pair, +) + + +class HitbtcOrderBookMessage(OrderBookMessage): + def __new__( + cls, + message_type: OrderBookMessageType, + content: Dict[str, any], + timestamp: Optional[float] = None, + *args, + **kwargs, + ): + if timestamp is None: + if message_type is OrderBookMessageType.SNAPSHOT: + raise ValueError("timestamp must not be None when initializing snapshot messages.") + timestamp = content["timestamp"] + + return super(HitbtcOrderBookMessage, cls).__new__( + cls, message_type, content, timestamp=timestamp, *args, **kwargs + ) + + @property + def update_id(self) -> int: + if self.type in [OrderBookMessageType.DIFF, OrderBookMessageType.SNAPSHOT]: + return int(self.timestamp * 1e3) + else: + return -1 + + @property + def trade_id(self) -> int: + if self.type is OrderBookMessageType.TRADE: + return int(self.timestamp * 1e3) + return -1 + + @property + def trading_pair(self) -> str: + if "trading_pair" in self.content: + return self.content["trading_pair"] + elif "symbol" in self.content: + return convert_from_exchange_trading_pair(self.content["symbol"]) + + @property + def asks(self) -> List[OrderBookRow]: + asks = map(self.content["ask"], lambda ask: {"price": ask["price"], "size": ask["size"]}) + + return [ + OrderBookRow(float(price), float(amount), self.update_id) for price, amount in asks + ] + + @property + def bids(self) -> List[OrderBookRow]: + bids = map(self.content["bid"], lambda bid: {"price": bid["price"], "size": bid["size"]}) + + return [ + OrderBookRow(float(price), float(amount), self.update_id) for price, amount in bids + ] + + def __eq__(self, other) -> bool: + return self.type == other.type and self.timestamp == other.timestamp + + def __lt__(self, other) -> bool: + if self.timestamp != other.timestamp: + return self.timestamp < other.timestamp + else: + """ + If timestamp is the same, the ordering is snapshot < diff < trade + """ + return self.type.value < other.type.value diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker.py b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker.py new file mode 100644 index 0000000000..d3161de17e --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +import asyncio +import bisect +import logging +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants +import time + +from collections import defaultdict, deque +from typing import Optional, Dict, List, Deque +from hummingbot.core.data_type.order_book_message import OrderBookMessageType +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.order_book_tracker import OrderBookTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_message import HitbtcOrderBookMessage +from hummingbot.connector.exchange.hitbtc.hitbtc_active_order_tracker import HitbtcActiveOrderTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_api_order_book_data_source import HitbtcAPIOrderBookDataSource +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book import HitbtcOrderBook + + +class HitbtcOrderBookTracker(OrderBookTracker): + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, trading_pairs: Optional[List[str]] = None,): + super().__init__(HitbtcAPIOrderBookDataSource(trading_pairs), trading_pairs) + + self._ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + self._order_book_snapshot_stream: asyncio.Queue = asyncio.Queue() + self._order_book_diff_stream: asyncio.Queue = asyncio.Queue() + self._order_book_trade_stream: asyncio.Queue = asyncio.Queue() + self._process_msg_deque_task: Optional[asyncio.Task] = None + self._past_diffs_windows: Dict[str, Deque] = {} + self._order_books: Dict[str, HitbtcOrderBook] = {} + self._saved_message_queues: Dict[str, Deque[HitbtcOrderBookMessage]] = \ + defaultdict(lambda: deque(maxlen=1000)) + self._active_order_trackers: Dict[str, HitbtcActiveOrderTracker] = defaultdict(HitbtcActiveOrderTracker) + self._order_book_stream_listener_task: Optional[asyncio.Task] = None + self._order_book_trade_listener_task: Optional[asyncio.Task] = None + + @property + def exchange_name(self) -> str: + """ + Name of the current exchange + """ + return Constants.EXCHANGE_NAME + + async def _track_single_book(self, trading_pair: str): + """ + Update an order book with changes from the latest batch of received messages + """ + past_diffs_window: Deque[HitbtcOrderBookMessage] = deque() + self._past_diffs_windows[trading_pair] = past_diffs_window + + message_queue: asyncio.Queue = self._tracking_message_queues[trading_pair] + order_book: HitbtcOrderBook = self._order_books[trading_pair] + active_order_tracker: HitbtcActiveOrderTracker = self._active_order_trackers[trading_pair] + + last_message_timestamp: float = time.time() + diff_messages_accepted: int = 0 + + while True: + try: + message: HitbtcOrderBookMessage = None + saved_messages: Deque[HitbtcOrderBookMessage] = self._saved_message_queues[trading_pair] + # Process saved messages first if there are any + if len(saved_messages) > 0: + message = saved_messages.popleft() + else: + message = await message_queue.get() + + if message.type is OrderBookMessageType.DIFF: + bids, asks = active_order_tracker.convert_diff_message_to_order_book_row(message) + order_book.apply_diffs(bids, asks, message.update_id) + past_diffs_window.append(message) + while len(past_diffs_window) > self.PAST_DIFF_WINDOW_SIZE: + past_diffs_window.popleft() + diff_messages_accepted += 1 + + # Output some statistics periodically. + now: float = time.time() + if int(now / 60.0) > int(last_message_timestamp / 60.0): + self.logger().debug(f"Processed {diff_messages_accepted} order book diffs for {trading_pair}.") + diff_messages_accepted = 0 + last_message_timestamp = now + elif message.type is OrderBookMessageType.SNAPSHOT: + past_diffs: List[HitbtcOrderBookMessage] = list(past_diffs_window) + # only replay diffs later than snapshot, first update active order with snapshot then replay diffs + replay_position = bisect.bisect_right(past_diffs, message) + replay_diffs = past_diffs[replay_position:] + s_bids, s_asks = active_order_tracker.convert_snapshot_message_to_order_book_row(message) + order_book.apply_snapshot(s_bids, s_asks, message.update_id) + for diff_message in replay_diffs: + d_bids, d_asks = active_order_tracker.convert_diff_message_to_order_book_row(diff_message) + order_book.apply_diffs(d_bids, d_asks, diff_message.update_id) + + self.logger().debug(f"Processed order book snapshot for {trading_pair}.") + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + f"Unexpected error processing order book messages for {trading_pair}.", + exc_info=True, + app_warning_msg="Unexpected error processing order book messages. Retrying after 5 seconds." + ) + await asyncio.sleep(5.0) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker_entry.py b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker_entry.py new file mode 100644 index 0000000000..5edfbadec0 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker_entry.py @@ -0,0 +1,21 @@ +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_tracker_entry import OrderBookTrackerEntry +from hummingbot.connector.exchange.hitbtc.hitbtc_active_order_tracker import HitbtcActiveOrderTracker + + +class HitbtcOrderBookTrackerEntry(OrderBookTrackerEntry): + def __init__( + self, trading_pair: str, timestamp: float, order_book: OrderBook, active_order_tracker: HitbtcActiveOrderTracker + ): + self._active_order_tracker = active_order_tracker + super(HitbtcOrderBookTrackerEntry, self).__init__(trading_pair, timestamp, order_book) + + def __repr__(self) -> str: + return ( + f"HitbtcOrderBookTrackerEntry(trading_pair='{self._trading_pair}', timestamp='{self._timestamp}', " + f"order_book='{self._order_book}')" + ) + + @property + def active_order_tracker(self) -> HitbtcActiveOrderTracker: + return self._active_order_tracker diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_user_stream_tracker.py b/hummingbot/connector/exchange/coinzoom/coinzoom_user_stream_tracker.py new file mode 100644 index 0000000000..7b04002ccd --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_user_stream_tracker.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +import asyncio +import logging +from typing import ( + Optional, + List, +) +from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.user_stream_tracker import ( + UserStreamTracker +) +from hummingbot.core.utils.async_utils import ( + safe_ensure_future, + safe_gather, +) +from hummingbot.connector.exchange.hitbtc.hitbtc_api_user_stream_data_source import \ + HitbtcAPIUserStreamDataSource +from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants + + +class HitbtcUserStreamTracker(UserStreamTracker): + _cbpust_logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._bust_logger is None: + cls._bust_logger = logging.getLogger(__name__) + return cls._bust_logger + + def __init__(self, + hitbtc_auth: Optional[HitbtcAuth] = None, + trading_pairs: Optional[List[str]] = []): + super().__init__() + self._hitbtc_auth: HitbtcAuth = hitbtc_auth + self._trading_pairs: List[str] = trading_pairs + self._ev_loop: asyncio.events.AbstractEventLoop = asyncio.get_event_loop() + self._data_source: Optional[UserStreamTrackerDataSource] = None + self._user_stream_tracking_task: Optional[asyncio.Task] = None + + @property + def data_source(self) -> UserStreamTrackerDataSource: + """ + *required + Initializes a user stream data source (user specific order diffs from live socket stream) + :return: OrderBookTrackerDataSource + """ + if not self._data_source: + self._data_source = HitbtcAPIUserStreamDataSource( + hitbtc_auth=self._hitbtc_auth, + trading_pairs=self._trading_pairs + ) + return self._data_source + + @property + def exchange_name(self) -> str: + """ + *required + Name of the current exchange + """ + return Constants.EXCHANGE_NAME + + async def start(self): + """ + *required + Start all listeners and tasks + """ + self._user_stream_tracking_task = safe_ensure_future( + self.data_source.listen_for_user_stream(self._ev_loop, self._user_stream) + ) + await safe_gather(self._user_stream_tracking_task) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py new file mode 100644 index 0000000000..c549ce8b72 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py @@ -0,0 +1,156 @@ +import aiohttp +import asyncio +import random +import re +from dateutil.parser import parse as dateparse +from typing import ( + Any, + Dict, + Optional, + Tuple, +) + +from hummingbot.core.utils.tracking_nonce import get_tracking_nonce +from hummingbot.client.config.config_var import ConfigVar +from hummingbot.client.config.config_methods import using_exchange +from .hitbtc_constants import Constants + + +TRADING_PAIR_SPLITTER = re.compile(Constants.TRADING_PAIR_SPLITTER) + +CENTRALIZED = True + +EXAMPLE_PAIR = "BTC-USD" + +DEFAULT_FEES = [0.1, 0.25] + + +class HitbtcAPIError(IOError): + def __init__(self, error_payload: Dict[str, Any]): + super().__init__(str(error_payload)) + self.error_payload = error_payload + + +# convert date string to timestamp +def str_date_to_ts(date: str) -> int: + return int(dateparse(date).timestamp()) + + +# Request ID class +class RequestId: + """ + Generate request ids + """ + _request_id: int = 0 + + @classmethod + def generate_request_id(cls) -> int: + return get_tracking_nonce() + + +def split_trading_pair(trading_pair: str) -> Optional[Tuple[str, str]]: + try: + m = TRADING_PAIR_SPLITTER.match(trading_pair) + return m.group(1), m.group(2) + # Exceptions are now logged as warnings in trading pair fetcher + except Exception: + return None + + +def convert_from_exchange_trading_pair(ex_trading_pair: str) -> Optional[str]: + regex_match = split_trading_pair(ex_trading_pair) + if regex_match is None: + return None + # HitBTC uses uppercase (BTCUSDT) + base_asset, quote_asset = split_trading_pair(ex_trading_pair) + return f"{base_asset.upper()}-{quote_asset.upper()}" + + +def convert_to_exchange_trading_pair(hb_trading_pair: str) -> str: + # HitBTC uses uppercase (BTCUSDT) + return hb_trading_pair.replace("-", "").upper() + + +def get_new_client_order_id(is_buy: bool, trading_pair: str) -> str: + side = "B" if is_buy else "S" + symbols = trading_pair.split("-") + base = symbols[0].upper() + quote = symbols[1].upper() + base_str = f"{base[0]}{base[-1]}" + quote_str = f"{quote[0]}{quote[-1]}" + return f"{Constants.HBOT_BROKER_ID}-{side}-{base_str}{quote_str}-{get_tracking_nonce()}" + + +def retry_sleep_time(try_count: int) -> float: + random.seed() + randSleep = 1 + float(random.randint(1, 10) / 100) + return float(2 + float(randSleep * (1 + (try_count ** try_count)))) + + +async def aiohttp_response_with_errors(request_coroutine): + http_status, parsed_response, request_errors = None, None, False + try: + async with request_coroutine as response: + http_status = response.status + try: + parsed_response = await response.json() + except Exception: + request_errors = True + try: + parsed_response = str(await response.read()) + if len(parsed_response) > 100: + parsed_response = f"{parsed_response[:100]} ... (truncated)" + except Exception: + pass + TempFailure = (parsed_response is None or + (response.status not in [200, 201] and "error" not in parsed_response)) + if TempFailure: + parsed_response = response.reason if parsed_response is None else parsed_response + request_errors = True + except Exception: + request_errors = True + return http_status, parsed_response, request_errors + + +async def api_call_with_retries(method, + endpoint, + params: Optional[Dict[str, Any]] = None, + shared_client=None, + try_count: int = 0) -> Dict[str, Any]: + url = f"{Constants.REST_URL}/{endpoint}" + headers = {"Content-Type": "application/json"} + http_client = shared_client if shared_client is not None else aiohttp.ClientSession() + # Build request coro + response_coro = http_client.request(method=method.upper(), url=url, headers=headers, + params=params, timeout=Constants.API_CALL_TIMEOUT) + http_status, parsed_response, request_errors = await aiohttp_response_with_errors(response_coro) + if shared_client is None: + await http_client.close() + if request_errors or parsed_response is None: + if try_count < Constants.API_MAX_RETRIES: + try_count += 1 + time_sleep = retry_sleep_time(try_count) + print(f"Error fetching data from {url}. HTTP status is {http_status}. " + f"Retrying in {time_sleep:.0f}s.") + await asyncio.sleep(time_sleep) + return await api_call_with_retries(method=method, endpoint=endpoint, params=params, + shared_client=shared_client, try_count=try_count) + else: + raise HitbtcAPIError({"error": parsed_response, "status": http_status}) + return parsed_response + + +KEYS = { + "hitbtc_api_key": + ConfigVar(key="hitbtc_api_key", + prompt=f"Enter your {Constants.EXCHANGE_NAME} API key >>> ", + required_if=using_exchange("hitbtc"), + is_secure=True, + is_connect_key=True), + "hitbtc_secret_key": + ConfigVar(key="hitbtc_secret_key", + prompt=f"Enter your {Constants.EXCHANGE_NAME} secret key >>> ", + required_if=using_exchange("hitbtc"), + is_secure=True, + is_connect_key=True), +} diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py b/hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py new file mode 100644 index 0000000000..da65b869a2 --- /dev/null +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +import asyncio +import copy +import logging +import websockets +import json +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants + + +from typing import ( + Any, + AsyncIterable, + Dict, + Optional, +) +from websockets.exceptions import ConnectionClosed +from hummingbot.logger import HummingbotLogger +from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth +from hummingbot.connector.exchange.hitbtc.hitbtc_utils import ( + RequestId, + HitbtcAPIError, +) + +# reusable websocket class +# ToDo: We should eventually remove this class, and instantiate web socket connection normally (see Binance for example) + + +class HitbtcWebsocket(RequestId): + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, + auth: Optional[HitbtcAuth] = None): + self._auth: Optional[HitbtcAuth] = auth + self._isPrivate = True if self._auth is not None else False + self._WS_URL = Constants.WS_PRIVATE_URL if self._isPrivate else Constants.WS_PUBLIC_URL + self._client: Optional[websockets.WebSocketClientProtocol] = None + + # connect to exchange + async def connect(self): + self._client = await websockets.connect(self._WS_URL) + + # if auth class was passed into websocket class + # we need to emit authenticated requests + if self._isPrivate: + auth_params = self._auth.generate_auth_dict_ws(self.generate_request_id()) + await self._emit("login", auth_params, no_id=True) + raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=Constants.MESSAGE_TIMEOUT) + json_msg = json.loads(raw_msg_str) + if json_msg.get("result") is not True: + err_msg = json_msg.get('error', {}).get('message') + raise HitbtcAPIError({"error": f"Failed to authenticate to websocket - {err_msg}."}) + + return self._client + + # disconnect from exchange + async def disconnect(self): + if self._client is None: + return + + await self._client.close() + + # receive & parse messages + async def _messages(self) -> AsyncIterable[Any]: + try: + while True: + try: + raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=Constants.MESSAGE_TIMEOUT) + try: + msg = json.loads(raw_msg_str) + # HitBTC doesn't support ping or heartbeat messages. + # Can handle them here if that changes - use `safe_ensure_future`. + yield msg + except ValueError: + continue + except asyncio.TimeoutError: + await asyncio.wait_for(self._client.ping(), timeout=Constants.PING_TIMEOUT) + except asyncio.TimeoutError: + self.logger().warning("WebSocket ping timed out. Going to reconnect...") + return + except ConnectionClosed: + return + finally: + await self.disconnect() + + # emit messages + async def _emit(self, method: str, data: Optional[Dict[str, Any]] = {}, no_id: bool = False) -> int: + id = self.generate_request_id() + + payload = { + "id": id, + "method": method, + "params": copy.deepcopy(data), + } + + await self._client.send(json.dumps(payload)) + + return id + + # request via websocket + async def request(self, method: str, data: Optional[Dict[str, Any]] = {}) -> int: + return await self._emit(method, data) + + # subscribe to a method + async def subscribe(self, + channel: str, + trading_pair: Optional[str] = None, + params: Optional[Dict[str, Any]] = {}) -> int: + if trading_pair is not None: + params['symbol'] = trading_pair + return await self.request(f"subscribe{channel}", params) + + # unsubscribe to a method + async def unsubscribe(self, + channel: str, + trading_pair: Optional[str] = None, + params: Optional[Dict[str, Any]] = {}) -> int: + if trading_pair is not None: + params['symbol'] = trading_pair + return await self.request(f"unsubscribe{channel}", params) + + # listen to messages by method + async def on_message(self) -> AsyncIterable[Any]: + async for msg in self._messages(): + yield msg diff --git a/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml b/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml index ed08e2fa90..0b381e0c06 100644 --- a/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml +++ b/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml @@ -17,6 +17,9 @@ beaxy_taker_fee: coinbase_pro_maker_fee: coinbase_pro_taker_fee: +coinzoom_maker_fee: +coinzoom_taker_fee: + dydx_maker_fee: dydx_taker_fee: diff --git a/hummingbot/templates/conf_global_TEMPLATE.yml b/hummingbot/templates/conf_global_TEMPLATE.yml index 765022d998..4424f82650 100644 --- a/hummingbot/templates/conf_global_TEMPLATE.yml +++ b/hummingbot/templates/conf_global_TEMPLATE.yml @@ -34,6 +34,9 @@ coinbase_pro_api_key: null coinbase_pro_secret_key: null coinbase_pro_passphrase: null +coinzoom_api_key: null +coinzoom_secret_key: null + dydx_eth_private_key: null dydx_node_address: null diff --git a/setup.py b/setup.py index 083859ad4c..8ba7d0e019 100755 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ def main(): "hummingbot.connector.exchange.bittrex", "hummingbot.connector.exchange.bamboo_relay", "hummingbot.connector.exchange.coinbase_pro", + "hummingbot.connector.exchange.coinzoom", "hummingbot.connector.exchange.dydx", "hummingbot.connector.exchange.huobi", "hummingbot.connector.exchange.radar_relay", diff --git a/test/connector/exchange/coinzoom/.gitignore b/test/connector/exchange/coinzoom/.gitignore new file mode 100644 index 0000000000..23d9952b8c --- /dev/null +++ b/test/connector/exchange/coinzoom/.gitignore @@ -0,0 +1 @@ +backups \ No newline at end of file diff --git a/test/connector/exchange/coinzoom/__init__.py b/test/connector/exchange/coinzoom/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/connector/exchange/coinzoom/test_coinzoom_auth.py b/test/connector/exchange/coinzoom/test_coinzoom_auth.py new file mode 100644 index 0000000000..6cc71c27e9 --- /dev/null +++ b/test/connector/exchange/coinzoom/test_coinzoom_auth.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +import sys +import asyncio +import unittest +import aiohttp +import conf +import logging +from os.path import join, realpath +from typing import Dict, Any +from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth +from hummingbot.connector.exchange.hitbtc.hitbtc_websocket import HitbtcWebsocket +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL +from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants + +sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +logging.basicConfig(level=METRICS_LOG_LEVEL) + + +class TestAuth(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + api_key = conf.hitbtc_api_key + secret_key = conf.hitbtc_secret_key + cls.auth = HitbtcAuth(api_key, secret_key) + + async def rest_auth(self) -> Dict[Any, Any]: + endpoint = Constants.ENDPOINT['USER_BALANCES'] + headers = self.auth.get_headers("GET", f"{Constants.REST_URL_AUTH}/{endpoint}", None) + response = await aiohttp.ClientSession().get(f"{Constants.REST_URL}/{endpoint}", headers=headers) + return await response.json() + + async def ws_auth(self) -> Dict[Any, Any]: + ws = HitbtcWebsocket(self.auth) + await ws.connect() + await ws.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"], None, {}) + async for response in ws.on_message(): + return response + + def test_rest_auth(self): + result = self.ev_loop.run_until_complete(self.rest_auth()) + if len(result) == 0 or "currency" not in result[0].keys(): + print(f"Unexpected response for API call: {result}") + assert "currency" in result[0].keys() + + def test_ws_auth(self): + try: + response = self.ev_loop.run_until_complete(self.ws_auth()) + no_errors = True + except Exception: + no_errors = False + assert no_errors is True + if 'result' not in response: + print(f"Unexpected response for API call: {response}") + assert response['result'] is True diff --git a/test/connector/exchange/coinzoom/test_coinzoom_exchange.py b/test/connector/exchange/coinzoom/test_coinzoom_exchange.py new file mode 100644 index 0000000000..0456f5a8a9 --- /dev/null +++ b/test/connector/exchange/coinzoom/test_coinzoom_exchange.py @@ -0,0 +1,438 @@ +from os.path import join, realpath +import sys; sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +import asyncio +import logging +from decimal import Decimal +import unittest +import contextlib +import time +import os +from typing import List +import conf +import math + +from hummingbot.core.clock import Clock, ClockMode +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL +from hummingbot.core.utils.async_utils import safe_gather, safe_ensure_future +from hummingbot.core.event.event_logger import EventLogger +from hummingbot.core.event.events import ( + BuyOrderCompletedEvent, + BuyOrderCreatedEvent, + MarketEvent, + OrderFilledEvent, + OrderType, + SellOrderCompletedEvent, + SellOrderCreatedEvent, + OrderCancelledEvent +) +from hummingbot.model.sql_connection_manager import ( + SQLConnectionManager, + SQLConnectionType +) +from hummingbot.model.market_state import MarketState +from hummingbot.model.order import Order +from hummingbot.model.trade_fill import TradeFill +from hummingbot.connector.markets_recorder import MarketsRecorder +from hummingbot.connector.exchange.hitbtc.hitbtc_exchange import HitbtcExchange + +logging.basicConfig(level=METRICS_LOG_LEVEL) + +API_KEY = conf.hitbtc_api_key +API_SECRET = conf.hitbtc_secret_key + + +class HitbtcExchangeUnitTest(unittest.TestCase): + events: List[MarketEvent] = [ + MarketEvent.BuyOrderCompleted, + MarketEvent.SellOrderCompleted, + MarketEvent.OrderFilled, + MarketEvent.TransactionFailure, + MarketEvent.BuyOrderCreated, + MarketEvent.SellOrderCreated, + MarketEvent.OrderCancelled, + MarketEvent.OrderFailure + ] + connector: HitbtcExchange + event_logger: EventLogger + trading_pair = "BTC-USD" + base_token, quote_token = trading_pair.split("-") + stack: contextlib.ExitStack + + @classmethod + def setUpClass(cls): + global MAINNET_RPC_URL + + cls.ev_loop = asyncio.get_event_loop() + + cls.clock: Clock = Clock(ClockMode.REALTIME) + cls.connector: HitbtcExchange = HitbtcExchange( + hitbtc_api_key=API_KEY, + hitbtc_secret_key=API_SECRET, + trading_pairs=[cls.trading_pair], + trading_required=True + ) + print("Initializing Hitbtc market... this will take about a minute.") + cls.clock.add_iterator(cls.connector) + cls.stack: contextlib.ExitStack = contextlib.ExitStack() + cls._clock = cls.stack.enter_context(cls.clock) + cls.ev_loop.run_until_complete(cls.wait_til_ready()) + print("Ready.") + + @classmethod + def tearDownClass(cls) -> None: + cls.stack.close() + + @classmethod + async def wait_til_ready(cls, connector = None): + if connector is None: + connector = cls.connector + while True: + now = time.time() + next_iteration = now // 1.0 + 1 + if connector.ready: + break + else: + await cls._clock.run_til(next_iteration) + await asyncio.sleep(1.0) + + def setUp(self): + self.db_path: str = realpath(join(__file__, "../connector_test.sqlite")) + try: + os.unlink(self.db_path) + except FileNotFoundError: + pass + + self.event_logger = EventLogger() + for event_tag in self.events: + self.connector.add_listener(event_tag, self.event_logger) + + def tearDown(self): + for event_tag in self.events: + self.connector.remove_listener(event_tag, self.event_logger) + self.event_logger = None + + async def run_parallel_async(self, *tasks): + future: asyncio.Future = safe_ensure_future(safe_gather(*tasks)) + while not future.done(): + now = time.time() + next_iteration = now // 1.0 + 1 + await self._clock.run_til(next_iteration) + await asyncio.sleep(1.0) + return future.result() + + def run_parallel(self, *tasks): + return self.ev_loop.run_until_complete(self.run_parallel_async(*tasks)) + + def _place_order(self, is_buy, amount, order_type, price, ex_order_id) -> str: + if is_buy: + cl_order_id = self.connector.buy(self.trading_pair, amount, order_type, price) + else: + cl_order_id = self.connector.sell(self.trading_pair, amount, order_type, price) + return cl_order_id + + def _cancel_order(self, cl_order_id, connector=None): + if connector is None: + connector = self.connector + return connector.cancel(self.trading_pair, cl_order_id) + + def test_estimate_fee(self): + maker_fee = self.connector.estimate_fee_pct(True) + self.assertAlmostEqual(maker_fee, Decimal("0.001")) + taker_fee = self.connector.estimate_fee_pct(False) + self.assertAlmostEqual(taker_fee, Decimal("0.0025")) + + def test_buy_and_sell(self): + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.02") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + quote_bal = self.connector.get_available_balance(self.quote_token) + base_bal = self.connector.get_available_balance(self.base_token) + + order_id = self._place_order(True, amount, OrderType.LIMIT, price, 1) + order_completed_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(5)) + trade_events = [t for t in self.event_logger.event_log if isinstance(t, OrderFilledEvent)] + base_amount_traded = sum(t.amount for t in trade_events) + quote_amount_traded = sum(t.amount * t.price for t in trade_events) + + self.assertTrue([evt.order_type == OrderType.LIMIT for evt in trade_events]) + self.assertEqual(order_id, order_completed_event.order_id) + self.assertEqual(amount, order_completed_event.base_asset_amount) + self.assertEqual("BTC", order_completed_event.base_asset) + self.assertEqual("USD", order_completed_event.quote_asset) + self.assertAlmostEqual(base_amount_traded, order_completed_event.base_asset_amount) + self.assertAlmostEqual(quote_amount_traded, order_completed_event.quote_asset_amount) + self.assertGreater(order_completed_event.fee_amount, Decimal(0)) + self.assertTrue(any([isinstance(event, BuyOrderCreatedEvent) and str(event.order_id) == str(order_id) + for event in self.event_logger.event_log])) + + # check available quote balance gets updated, we need to wait a bit for the balance message to arrive + expected_quote_bal = quote_bal - quote_amount_traded + # self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.ev_loop.run_until_complete(self.connector._update_balances()) + self.assertAlmostEqual(expected_quote_bal, self.connector.get_available_balance(self.quote_token), 1) + + # Reset the logs + self.event_logger.clear() + + # Try to sell back the same amount to the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.98") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + order_id = self._place_order(False, amount, OrderType.LIMIT, price, 2) + order_completed_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCompletedEvent)) + trade_events = [t for t in self.event_logger.event_log if isinstance(t, OrderFilledEvent)] + base_amount_traded = sum(t.amount for t in trade_events) + quote_amount_traded = sum(t.amount * t.price for t in trade_events) + + self.assertTrue([evt.order_type == OrderType.LIMIT for evt in trade_events]) + self.assertEqual(order_id, order_completed_event.order_id) + self.assertEqual(amount, order_completed_event.base_asset_amount) + self.assertEqual("BTC", order_completed_event.base_asset) + self.assertEqual("USD", order_completed_event.quote_asset) + self.assertAlmostEqual(base_amount_traded, order_completed_event.base_asset_amount) + self.assertAlmostEqual(quote_amount_traded, order_completed_event.quote_asset_amount) + self.assertGreater(order_completed_event.fee_amount, Decimal(0)) + self.assertTrue(any([isinstance(event, SellOrderCreatedEvent) and event.order_id == order_id + for event in self.event_logger.event_log])) + + # check available base balance gets updated, we need to wait a bit for the balance message to arrive + expected_base_bal = base_bal + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.ev_loop.run_until_complete(self.connector._update_balances()) + self.ev_loop.run_until_complete(asyncio.sleep(5)) + self.assertAlmostEqual(expected_base_bal, self.connector.get_available_balance(self.base_token), 5) + + def test_limit_makers_unfilled(self): + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.8") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.ev_loop.run_until_complete(self.connector._update_balances()) + self.ev_loop.run_until_complete(asyncio.sleep(2)) + quote_bal = self.connector.get_available_balance(self.quote_token) + + cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + # check available quote balance gets updated, we need to wait a bit for the balance message to arrive + taker_fee = self.connector.estimate_fee_pct(False) + quote_amount = ((price * amount)) + quote_amount = ((price * amount) * (Decimal("1") + taker_fee)) + expected_quote_bal = quote_bal - quote_amount + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.ev_loop.run_until_complete(self.connector._update_balances()) + self.ev_loop.run_until_complete(asyncio.sleep(2)) + + self.assertAlmostEqual(expected_quote_bal, self.connector.get_available_balance(self.quote_token), 5) + self._cancel_order(cl_order_id) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.2") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + cl_order_id = self._place_order(False, amount, OrderType.LIMIT_MAKER, price, 2) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + self._cancel_order(cl_order_id) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + # # @TODO: find a way to create "rejected" + # def test_limit_maker_rejections(self): + # price = self.connector.get_price(self.trading_pair, True) * Decimal("1.2") + # price = self.connector.quantize_order_price(self.trading_pair, price) + # amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.000001")) + # cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1) + # event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + # self.assertEqual(cl_order_id, event.order_id) + + # price = self.connector.get_price(self.trading_pair, False) * Decimal("0.8") + # price = self.connector.quantize_order_price(self.trading_pair, price) + # amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.000001")) + # cl_order_id = self._place_order(False, amount, OrderType.LIMIT_MAKER, price, 2) + # event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + # self.assertEqual(cl_order_id, event.order_id) + + def test_cancel_all(self): + bid_price = self.connector.get_price(self.trading_pair, True) + ask_price = self.connector.get_price(self.trading_pair, False) + bid_price = self.connector.quantize_order_price(self.trading_pair, bid_price * Decimal("0.9")) + ask_price = self.connector.quantize_order_price(self.trading_pair, ask_price * Decimal("1.1")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + buy_id = self._place_order(True, amount, OrderType.LIMIT, bid_price, 1) + sell_id = self._place_order(False, amount, OrderType.LIMIT, ask_price, 2) + + self.ev_loop.run_until_complete(asyncio.sleep(1)) + asyncio.ensure_future(self.connector.cancel_all(5)) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + cancel_events = [t for t in self.event_logger.event_log if isinstance(t, OrderCancelledEvent)] + self.assertEqual({buy_id, sell_id}, {o.order_id for o in cancel_events}) + + def test_order_quantized_values(self): + bid_price: Decimal = self.connector.get_price(self.trading_pair, True) + ask_price: Decimal = self.connector.get_price(self.trading_pair, False) + mid_price: Decimal = (bid_price + ask_price) / 2 + + # Make sure there's enough balance to make the limit orders. + self.assertGreater(self.connector.get_balance("BTC"), Decimal("0.0005")) + self.assertGreater(self.connector.get_balance("USD"), Decimal("10")) + + # Intentionally set some prices with too many decimal places s.t. they + # need to be quantized. Also, place them far away from the mid-price s.t. they won't + # get filled during the test. + bid_price = self.connector.quantize_order_price(self.trading_pair, mid_price * Decimal("0.9333192292111341")) + ask_price = self.connector.quantize_order_price(self.trading_pair, mid_price * Decimal("1.1492431474884933")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.000123456")) + + # Test bid order + cl_order_id_1 = self._place_order(True, amount, OrderType.LIMIT, bid_price, 1) + # Wait for the order created event and examine the order made + self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + + # Test ask order + cl_order_id_2 = self._place_order(False, amount, OrderType.LIMIT, ask_price, 1) + # Wait for the order created event and examine and order made + self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCreatedEvent)) + + self._cancel_order(cl_order_id_1) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self._cancel_order(cl_order_id_2) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + + def test_orders_saving_and_restoration(self): + config_path = "test_config" + strategy_name = "test_strategy" + sql = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, db_path=self.db_path) + order_id = None + recorder = MarketsRecorder(sql, [self.connector], config_path, strategy_name) + recorder.start() + + try: + self.connector._in_flight_orders.clear() + self.assertEqual(0, len(self.connector.tracking_states)) + + # Try to put limit buy order for 0.02 ETH worth of ZRX, and watch for order creation event. + current_bid_price: Decimal = self.connector.get_price(self.trading_pair, True) + price: Decimal = current_bid_price * Decimal("0.8") + price = self.connector.quantize_order_price(self.trading_pair, price) + + amount: Decimal = Decimal("0.0001") + amount = self.connector.quantize_order_amount(self.trading_pair, amount) + + cl_order_id = self._place_order(True, amount, OrderType.LIMIT_MAKER, price, 1) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + + # Verify tracking states + self.assertEqual(1, len(self.connector.tracking_states)) + self.assertEqual(cl_order_id, list(self.connector.tracking_states.keys())[0]) + + # Verify orders from recorder + recorded_orders: List[Order] = recorder.get_orders_for_config_and_market(config_path, self.connector) + self.assertEqual(1, len(recorded_orders)) + self.assertEqual(cl_order_id, recorded_orders[0].id) + + # Verify saved market states + saved_market_states: MarketState = recorder.get_market_states(config_path, self.connector) + self.assertIsNotNone(saved_market_states) + self.assertIsInstance(saved_market_states.saved_state, dict) + self.assertGreater(len(saved_market_states.saved_state), 0) + + # Close out the current market and start another market. + self.connector.stop(self._clock) + self.ev_loop.run_until_complete(asyncio.sleep(5)) + self.clock.remove_iterator(self.connector) + for event_tag in self.events: + self.connector.remove_listener(event_tag, self.event_logger) + # Clear the event loop + self.event_logger.clear() + new_connector = HitbtcExchange(API_KEY, API_SECRET, [self.trading_pair], True) + for event_tag in self.events: + new_connector.add_listener(event_tag, self.event_logger) + recorder.stop() + recorder = MarketsRecorder(sql, [new_connector], config_path, strategy_name) + recorder.start() + saved_market_states = recorder.get_market_states(config_path, new_connector) + self.clock.add_iterator(new_connector) + self.ev_loop.run_until_complete(self.wait_til_ready(new_connector)) + self.assertEqual(0, len(new_connector.limit_orders)) + self.assertEqual(0, len(new_connector.tracking_states)) + new_connector.restore_tracking_states(saved_market_states.saved_state) + self.assertEqual(1, len(new_connector.limit_orders)) + self.assertEqual(1, len(new_connector.tracking_states)) + + # Cancel the order and verify that the change is saved. + self._cancel_order(cl_order_id, new_connector) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + recorder.save_market_states(config_path, new_connector) + order_id = None + self.assertEqual(0, len(new_connector.limit_orders)) + self.assertEqual(0, len(new_connector.tracking_states)) + saved_market_states = recorder.get_market_states(config_path, new_connector) + self.assertEqual(0, len(saved_market_states.saved_state)) + finally: + if order_id is not None: + self.connector.cancel(self.trading_pair, cl_order_id) + self.run_parallel(self.event_logger.wait_for(OrderCancelledEvent)) + + recorder.stop() + os.unlink(self.db_path) + + def test_update_last_prices(self): + # This is basic test to see if order_book last_trade_price is initiated and updated. + for order_book in self.connector.order_books.values(): + for _ in range(5): + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.assertFalse(math.isnan(order_book.last_trade_price)) + + def test_filled_orders_recorded(self): + config_path: str = "test_config" + strategy_name: str = "test_strategy" + sql = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, db_path=self.db_path) + order_id = None + recorder = MarketsRecorder(sql, [self.connector], config_path, strategy_name) + recorder.start() + + try: + # Try to buy some token from the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.05") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + order_id = self._place_order(True, amount, OrderType.LIMIT, price, 1) + self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + + # Reset the logs + self.event_logger.clear() + + # Try to sell back the same amount to the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.95") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + order_id = self._place_order(False, amount, OrderType.LIMIT, price, 2) + self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + + # Query the persisted trade logs + trade_fills: List[TradeFill] = recorder.get_trades_for_config(config_path) + self.assertGreaterEqual(len(trade_fills), 2) + buy_fills: List[TradeFill] = [t for t in trade_fills if t.trade_type == "BUY"] + sell_fills: List[TradeFill] = [t for t in trade_fills if t.trade_type == "SELL"] + self.assertGreaterEqual(len(buy_fills), 1) + self.assertGreaterEqual(len(sell_fills), 1) + + order_id = None + + finally: + if order_id is not None: + self.connector.cancel(self.trading_pair, order_id) + self.run_parallel(self.event_logger.wait_for(OrderCancelledEvent)) + + recorder.stop() + os.unlink(self.db_path) diff --git a/test/connector/exchange/coinzoom/test_coinzoom_order_book_tracker.py b/test/connector/exchange/coinzoom/test_coinzoom_order_book_tracker.py new file mode 100755 index 0000000000..ae3778e7c9 --- /dev/null +++ b/test/connector/exchange/coinzoom/test_coinzoom_order_book_tracker.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +import sys +import math +import time +import asyncio +import logging +import unittest +from os.path import join, realpath +from typing import Dict, Optional, List +from hummingbot.core.event.event_logger import EventLogger +from hummingbot.core.event.events import OrderBookEvent, OrderBookTradeEvent, TradeType +from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_tracker import HitbtcOrderBookTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_api_order_book_data_source import HitbtcAPIOrderBookDataSource +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL + + +sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +logging.basicConfig(level=METRICS_LOG_LEVEL) + + +class HitbtcOrderBookTrackerUnitTest(unittest.TestCase): + order_book_tracker: Optional[HitbtcOrderBookTracker] = None + events: List[OrderBookEvent] = [ + OrderBookEvent.TradeEvent + ] + trading_pairs: List[str] = [ + "BTC-USD", + "ETH-USD", + ] + + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + cls.order_book_tracker: HitbtcOrderBookTracker = HitbtcOrderBookTracker(cls.trading_pairs) + cls.order_book_tracker.start() + cls.ev_loop.run_until_complete(cls.wait_til_tracker_ready()) + + @classmethod + async def wait_til_tracker_ready(cls): + while True: + if len(cls.order_book_tracker.order_books) > 0: + print("Initialized real-time order books.") + return + await asyncio.sleep(1) + + async def run_parallel_async(self, *tasks, timeout=None): + future: asyncio.Future = asyncio.ensure_future(asyncio.gather(*tasks)) + timer = 0 + while not future.done(): + if timeout and timer > timeout: + raise Exception("Timeout running parallel async tasks in tests") + timer += 1 + now = time.time() + _next_iteration = now // 1.0 + 1 # noqa: F841 + await asyncio.sleep(1.0) + return future.result() + + def run_parallel(self, *tasks): + return self.ev_loop.run_until_complete(self.run_parallel_async(*tasks)) + + def setUp(self): + self.event_logger = EventLogger() + for event_tag in self.events: + for trading_pair, order_book in self.order_book_tracker.order_books.items(): + order_book.add_listener(event_tag, self.event_logger) + + def test_order_book_trade_event_emission(self): + """ + Tests if the order book tracker is able to retrieve order book trade message from exchange and emit order book + trade events after correctly parsing the trade messages + """ + self.run_parallel(self.event_logger.wait_for(OrderBookTradeEvent)) + for ob_trade_event in self.event_logger.event_log: + self.assertTrue(type(ob_trade_event) == OrderBookTradeEvent) + self.assertTrue(ob_trade_event.trading_pair in self.trading_pairs) + self.assertTrue(type(ob_trade_event.timestamp) in [float, int]) + self.assertTrue(type(ob_trade_event.amount) == float) + self.assertTrue(type(ob_trade_event.price) == float) + self.assertTrue(type(ob_trade_event.type) == TradeType) + # datetime is in seconds + self.assertTrue(math.ceil(math.log10(ob_trade_event.timestamp)) == 10) + self.assertTrue(ob_trade_event.amount > 0) + self.assertTrue(ob_trade_event.price > 0) + + def test_tracker_integrity(self): + # Wait 5 seconds to process some diffs. + self.ev_loop.run_until_complete(asyncio.sleep(5.0)) + order_books: Dict[str, OrderBook] = self.order_book_tracker.order_books + eth_usd: OrderBook = order_books["ETH-USD"] + self.assertIsNot(eth_usd.last_diff_uid, 0) + self.assertGreaterEqual(eth_usd.get_price_for_volume(True, 10).result_price, + eth_usd.get_price(True)) + self.assertLessEqual(eth_usd.get_price_for_volume(False, 10).result_price, + eth_usd.get_price(False)) + + def test_api_get_last_traded_prices(self): + prices = self.ev_loop.run_until_complete( + HitbtcAPIOrderBookDataSource.get_last_traded_prices(["BTC-USD", "LTC-BTC"])) + for key, value in prices.items(): + print(f"{key} last_trade_price: {value}") + self.assertGreater(prices["BTC-USD"], 1000) + self.assertLess(prices["LTC-BTC"], 1) diff --git a/test/connector/exchange/coinzoom/test_coinzoom_user_stream_tracker.py b/test/connector/exchange/coinzoom/test_coinzoom_user_stream_tracker.py new file mode 100644 index 0000000000..5c82f2372b --- /dev/null +++ b/test/connector/exchange/coinzoom/test_coinzoom_user_stream_tracker.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +import sys +import asyncio +import logging +import unittest +import conf + +from os.path import join, realpath +from hummingbot.connector.exchange.hitbtc.hitbtc_user_stream_tracker import HitbtcUserStreamTracker +from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth +from hummingbot.core.utils.async_utils import safe_ensure_future +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL + + +sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +logging.basicConfig(level=METRICS_LOG_LEVEL) + + +class HitbtcUserStreamTrackerUnitTest(unittest.TestCase): + api_key = conf.hitbtc_api_key + api_secret = conf.hitbtc_secret_key + + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + cls.trading_pairs = ["BTC-USD"] + cls.user_stream_tracker: HitbtcUserStreamTracker = HitbtcUserStreamTracker( + hitbtc_auth=HitbtcAuth(cls.api_key, cls.api_secret), + trading_pairs=cls.trading_pairs) + cls.user_stream_tracker_task: asyncio.Task = safe_ensure_future(cls.user_stream_tracker.start()) + + def test_user_stream(self): + # Wait process some msgs. + print("Sleeping for 30s to gather some user stream messages.") + self.ev_loop.run_until_complete(asyncio.sleep(30.0)) + print(self.user_stream_tracker.user_stream) From 1f8bdb69ae210f270f5940e2e4961450f6962de9 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Sat, 20 Mar 2021 03:07:08 +0000 Subject: [PATCH 02/23] CoinZoom: Renames --- .../coinzoom_active_order_tracker.pxd | 2 +- .../coinzoom_active_order_tracker.pyx | 12 ++-- .../coinzoom_api_order_book_data_source.py | 34 +++++----- .../coinzoom_api_user_stream_data_source.py | 20 +++--- .../exchange/coinzoom/coinzoom_auth.py | 8 +-- .../exchange/coinzoom/coinzoom_constants.py | 8 +-- .../exchange/coinzoom/coinzoom_exchange.py | 62 +++++++++---------- .../coinzoom/coinzoom_in_flight_order.py | 4 +- .../exchange/coinzoom/coinzoom_order_book.py | 30 ++++----- .../coinzoom/coinzoom_order_book_message.py | 6 +- .../coinzoom/coinzoom_order_book_tracker.py | 32 +++++----- .../coinzoom_order_book_tracker_entry.py | 12 ++-- .../coinzoom/coinzoom_user_stream_tracker.py | 18 +++--- .../exchange/coinzoom/coinzoom_utils.py | 22 +++---- .../exchange/coinzoom/coinzoom_websocket.py | 18 +++--- .../exchange/coinzoom/test_coinzoom_auth.py | 14 ++--- .../coinzoom/test_coinzoom_exchange.py | 20 +++--- .../test_coinzoom_order_book_tracker.py | 12 ++-- .../test_coinzoom_user_stream_tracker.py | 14 ++--- 19 files changed, 174 insertions(+), 174 deletions(-) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd index 97c2af5b02..7990aaf2aa 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd @@ -1,7 +1,7 @@ # distutils: language=c++ cimport numpy as np -cdef class HitbtcActiveOrderTracker: +cdef class CoinzoomActiveOrderTracker: cdef dict _active_bids cdef dict _active_asks diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx index bef95bea6d..8e4bb48d4a 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx @@ -9,12 +9,12 @@ from hummingbot.core.data_type.order_book_row import OrderBookRow _logger = None s_empty_diff = np.ndarray(shape=(0, 4), dtype="float64") -HitbtcOrderBookTrackingDictionary = Dict[Decimal, Dict[str, Dict[str, any]]] +CoinzoomOrderBookTrackingDictionary = Dict[Decimal, Dict[str, Dict[str, any]]] -cdef class HitbtcActiveOrderTracker: +cdef class CoinzoomActiveOrderTracker: def __init__(self, - active_asks: HitbtcOrderBookTrackingDictionary = None, - active_bids: HitbtcOrderBookTrackingDictionary = None): + active_asks: CoinzoomOrderBookTrackingDictionary = None, + active_bids: CoinzoomOrderBookTrackingDictionary = None): super().__init__() self._active_asks = active_asks or {} self._active_bids = active_bids or {} @@ -27,11 +27,11 @@ cdef class HitbtcActiveOrderTracker: return _logger @property - def active_asks(self) -> HitbtcOrderBookTrackingDictionary: + def active_asks(self) -> CoinzoomOrderBookTrackingDictionary: return self._active_asks @property - def active_bids(self) -> HitbtcOrderBookTrackingDictionary: + def active_bids(self) -> CoinzoomOrderBookTrackingDictionary: return self._active_bids # TODO: research this more diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_api_order_book_data_source.py b/hummingbot/connector/exchange/coinzoom/coinzoom_api_order_book_data_source.py index 40d83516da..56542705ac 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_api_order_book_data_source.py @@ -9,20 +9,20 @@ from hummingbot.core.data_type.order_book_message import OrderBookMessage from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource from hummingbot.logger import HummingbotLogger -from .hitbtc_constants import Constants -from .hitbtc_active_order_tracker import HitbtcActiveOrderTracker -from .hitbtc_order_book import HitbtcOrderBook -from .hitbtc_websocket import HitbtcWebsocket -from .hitbtc_utils import ( +from .coinzoom_constants import Constants +from .coinzoom_active_order_tracker import CoinzoomActiveOrderTracker +from .coinzoom_order_book import CoinzoomOrderBook +from .coinzoom_websocket import CoinzoomWebsocket +from .coinzoom_utils import ( str_date_to_ts, convert_to_exchange_trading_pair, convert_from_exchange_trading_pair, api_call_with_retries, - HitbtcAPIError, + CoinzoomAPIError, ) -class HitbtcAPIOrderBookDataSource(OrderBookTrackerDataSource): +class CoinzoomAPIOrderBookDataSource(OrderBookTrackerDataSource): _logger: Optional[HummingbotLogger] = None @classmethod @@ -59,7 +59,7 @@ async def fetch_trading_pairs() -> List[str]: # Filter out unmatched pairs so nothing breaks return [sym for sym in trading_pairs if sym is not None] except Exception: - # Do nothing if the request fails -- there will be no autocomplete for HitBTC trading pairs + # Do nothing if the request fails -- there will be no autocomplete for CoinZoom trading pairs pass return [] @@ -73,7 +73,7 @@ async def get_order_book_data(trading_pair: str) -> Dict[str, any]: orderbook_response: Dict[Any] = await api_call_with_retries("GET", Constants.ENDPOINT["ORDER_BOOK"], params={"limit": 150, "symbols": ex_pair}) return orderbook_response[ex_pair] - except HitbtcAPIError as e: + except CoinzoomAPIError as e: err = e.error_payload.get('error', e.error_payload) raise IOError( f"Error fetching OrderBook for {trading_pair} at {Constants.EXCHANGE_NAME}. " @@ -82,12 +82,12 @@ async def get_order_book_data(trading_pair: str) -> Dict[str, any]: async def get_new_order_book(self, trading_pair: str) -> OrderBook: snapshot: Dict[str, Any] = await self.get_order_book_data(trading_pair) snapshot_timestamp: float = time.time() - snapshot_msg: OrderBookMessage = HitbtcOrderBook.snapshot_message_from_exchange( + snapshot_msg: OrderBookMessage = CoinzoomOrderBook.snapshot_message_from_exchange( snapshot, snapshot_timestamp, metadata={"trading_pair": trading_pair}) order_book = self.order_book_create_function() - active_order_tracker: HitbtcActiveOrderTracker = HitbtcActiveOrderTracker() + active_order_tracker: CoinzoomActiveOrderTracker = CoinzoomActiveOrderTracker() bids, asks = active_order_tracker.convert_snapshot_message_to_order_book_row(snapshot_msg) order_book.apply_snapshot(bids, asks, snapshot_msg.update_id) return order_book @@ -98,7 +98,7 @@ async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asynci """ while True: try: - ws = HitbtcWebsocket() + ws = CoinzoomWebsocket() await ws.connect() for pair in self._trading_pairs: @@ -116,7 +116,7 @@ async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asynci for trade in trades_data["data"]: trade: Dict[Any] = trade trade_timestamp: int = str_date_to_ts(trade["timestamp"]) - trade_msg: OrderBookMessage = HitbtcOrderBook.trade_message_from_exchange( + trade_msg: OrderBookMessage = CoinzoomOrderBook.trade_message_from_exchange( trade, trade_timestamp, metadata={"trading_pair": pair}) @@ -136,7 +136,7 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp """ while True: try: - ws = HitbtcWebsocket() + ws = CoinzoomWebsocket() await ws.connect() order_book_methods = [ @@ -157,9 +157,9 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp timestamp: int = str_date_to_ts(order_book_data["timestamp"]) pair: str = convert_from_exchange_trading_pair(order_book_data["symbol"]) - order_book_msg_cls = (HitbtcOrderBook.diff_message_from_exchange + order_book_msg_cls = (CoinzoomOrderBook.diff_message_from_exchange if method == Constants.WS_METHODS['ORDERS_UPDATE'] else - HitbtcOrderBook.snapshot_message_from_exchange) + CoinzoomOrderBook.snapshot_message_from_exchange) orderbook_msg: OrderBookMessage = order_book_msg_cls( order_book_data, @@ -188,7 +188,7 @@ async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, try: snapshot: Dict[str, any] = await self.get_order_book_data(trading_pair) snapshot_timestamp: int = str_date_to_ts(snapshot["timestamp"]) - snapshot_msg: OrderBookMessage = HitbtcOrderBook.snapshot_message_from_exchange( + snapshot_msg: OrderBookMessage = CoinzoomOrderBook.snapshot_message_from_exchange( snapshot, snapshot_timestamp, metadata={"trading_pair": trading_pair} diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py b/hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py index 954ab9c344..38d7b704ff 100755 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py @@ -10,13 +10,13 @@ ) from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource from hummingbot.logger import HummingbotLogger -from .hitbtc_constants import Constants -from .hitbtc_auth import HitbtcAuth -from .hitbtc_utils import HitbtcAPIError -from .hitbtc_websocket import HitbtcWebsocket +from .coinzoom_constants import Constants +from .coinzoom_auth import CoinzoomAuth +from .coinzoom_utils import CoinzoomAPIError +from .coinzoom_websocket import CoinzoomWebsocket -class HitbtcAPIUserStreamDataSource(UserStreamTrackerDataSource): +class CoinzoomAPIUserStreamDataSource(UserStreamTrackerDataSource): _logger: Optional[HummingbotLogger] = None @@ -26,9 +26,9 @@ def logger(cls) -> HummingbotLogger: cls._logger = logging.getLogger(__name__) return cls._logger - def __init__(self, hitbtc_auth: HitbtcAuth, trading_pairs: Optional[List[str]] = []): - self._hitbtc_auth: HitbtcAuth = hitbtc_auth - self._ws: HitbtcWebsocket = None + def __init__(self, coinzoom_auth: CoinzoomAuth, trading_pairs: Optional[List[str]] = []): + self._coinzoom_auth: CoinzoomAuth = coinzoom_auth + self._ws: CoinzoomWebsocket = None self._trading_pairs = trading_pairs self._current_listen_key = None self._listen_for_user_stream_task = None @@ -48,7 +48,7 @@ async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: """ try: - self._ws = HitbtcWebsocket(self._hitbtc_auth) + self._ws = CoinzoomWebsocket(self._coinzoom_auth) await self._ws.connect() @@ -87,7 +87,7 @@ async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: a output.put_nowait(msg) except asyncio.CancelledError: raise - except HitbtcAPIError as e: + except CoinzoomAPIError as e: self.logger().error(e.error_payload.get('error'), exc_info=True) raise except Exception: diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_auth.py b/hummingbot/connector/exchange/coinzoom/coinzoom_auth.py index be37f2e149..550b1dd6e9 100755 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_auth.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_auth.py @@ -5,9 +5,9 @@ from typing import Dict, Any -class HitbtcAuth(): +class CoinzoomAuth(): """ - Auth class required by HitBTC API + Auth class required by CoinZoom API Learn more at https://exchange-docs.crypto.com/#digital-signature """ def __init__(self, api_key: str, secret_key: str): @@ -44,7 +44,7 @@ def generate_payload( def generate_auth_dict_ws(self, nonce: int): """ - Generates an authentication params for HitBTC websockets login + Generates an authentication params for CoinZoom websockets login :return: a dictionary of auth params """ return { @@ -61,7 +61,7 @@ def get_headers(self, url, params) -> Dict[str, Any]: """ - Generates authentication headers required by HitBTC + Generates authentication headers required by CoinZoom :return: a dictionary of auth headers """ payload = self.generate_payload(method, url, params) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py index 538e0b21f2..43d69678b4 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py @@ -1,10 +1,10 @@ # A single source of truth for constant variables related to the exchange class Constants: - EXCHANGE_NAME = "hitbtc" - REST_URL = "https://api.hitbtc.com/api/2" + EXCHANGE_NAME = "coinzoom" + REST_URL = "https://api.coinzoom.com/api/2" REST_URL_AUTH = "/api/2" - WS_PRIVATE_URL = "wss://api.hitbtc.com/api/2/ws/trading" - WS_PUBLIC_URL = "wss://api.hitbtc.com/api/2/ws/public" + WS_PRIVATE_URL = "wss://api.coinzoom.com/api/2/ws/trading" + WS_PUBLIC_URL = "wss://api.coinzoom.com/api/2/ws/public" HBOT_BROKER_ID = "refzzz48" diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py index 9f6f83ec15..d9eaf36f71 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py @@ -35,28 +35,28 @@ TradeFee ) from hummingbot.connector.exchange_base import ExchangeBase -from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_tracker import HitbtcOrderBookTracker -from hummingbot.connector.exchange.hitbtc.hitbtc_user_stream_tracker import HitbtcUserStreamTracker -from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth -from hummingbot.connector.exchange.hitbtc.hitbtc_in_flight_order import HitbtcInFlightOrder -from hummingbot.connector.exchange.hitbtc.hitbtc_utils import ( +from hummingbot.connector.exchange.coinzoom.coinzoom_order_book_tracker import CoinzoomOrderBookTracker +from hummingbot.connector.exchange.coinzoom.coinzoom_user_stream_tracker import CoinzoomUserStreamTracker +from hummingbot.connector.exchange.coinzoom.coinzoom_auth import CoinzoomAuth +from hummingbot.connector.exchange.coinzoom.coinzoom_in_flight_order import CoinzoomInFlightOrder +from hummingbot.connector.exchange.coinzoom.coinzoom_utils import ( convert_from_exchange_trading_pair, convert_to_exchange_trading_pair, get_new_client_order_id, aiohttp_response_with_errors, retry_sleep_time, str_date_to_ts, - HitbtcAPIError, + CoinzoomAPIError, ) -from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants +from hummingbot.connector.exchange.coinzoom.coinzoom_constants import Constants from hummingbot.core.data_type.common import OpenOrder ctce_logger = None s_decimal_NaN = Decimal("nan") -class HitbtcExchange(ExchangeBase): +class CoinzoomExchange(ExchangeBase): """ - HitbtcExchange connects with HitBTC exchange and provides order book pricing, user account tracking and + CoinzoomExchange connects with CoinZoom exchange and provides order book pricing, user account tracking and trading functionality. """ ORDER_NOT_EXIST_CONFIRMATION_COUNT = 3 @@ -70,28 +70,28 @@ def logger(cls) -> HummingbotLogger: return ctce_logger def __init__(self, - hitbtc_api_key: str, - hitbtc_secret_key: str, + coinzoom_api_key: str, + coinzoom_secret_key: str, trading_pairs: Optional[List[str]] = None, trading_required: bool = True ): """ - :param hitbtc_api_key: The API key to connect to private HitBTC APIs. - :param hitbtc_secret_key: The API secret. + :param coinzoom_api_key: The API key to connect to private CoinZoom APIs. + :param coinzoom_secret_key: The API secret. :param trading_pairs: The market trading pairs which to track order book data. :param trading_required: Whether actual trading is needed. """ super().__init__() self._trading_required = trading_required self._trading_pairs = trading_pairs - self._hitbtc_auth = HitbtcAuth(hitbtc_api_key, hitbtc_secret_key) - self._order_book_tracker = HitbtcOrderBookTracker(trading_pairs=trading_pairs) - self._user_stream_tracker = HitbtcUserStreamTracker(self._hitbtc_auth, trading_pairs) + self._coinzoom_auth = CoinzoomAuth(coinzoom_api_key, coinzoom_secret_key) + self._order_book_tracker = CoinzoomOrderBookTracker(trading_pairs=trading_pairs) + self._user_stream_tracker = CoinzoomUserStreamTracker(self._coinzoom_auth, trading_pairs) self._ev_loop = asyncio.get_event_loop() self._shared_client = None self._poll_notifier = asyncio.Event() self._last_timestamp = 0 - self._in_flight_orders = {} # Dict[client_order_id:str, HitbtcInFlightOrder] + self._in_flight_orders = {} # Dict[client_order_id:str, CoinzoomInFlightOrder] self._order_not_found_records = {} # Dict[client_order_id:str, count:int] self._trading_rules = {} # Dict[trading_pair:str, TradingRule] self._status_polling_task = None @@ -101,7 +101,7 @@ def __init__(self, @property def name(self) -> str: - return "hitbtc" + return "coinzoom" @property def order_books(self) -> Dict[str, OrderBook]: @@ -112,7 +112,7 @@ def trading_rules(self) -> Dict[str, TradingRule]: return self._trading_rules @property - def in_flight_orders(self) -> Dict[str, HitbtcInFlightOrder]: + def in_flight_orders(self) -> Dict[str, CoinzoomInFlightOrder]: return self._in_flight_orders @property @@ -161,7 +161,7 @@ def restore_tracking_states(self, saved_states: Dict[str, any]): :param saved_states: The saved tracking_states. """ self._in_flight_orders.update({ - key: HitbtcInFlightOrder.from_json(value) + key: CoinzoomInFlightOrder.from_json(value) for key, value in saved_states.items() }) @@ -322,8 +322,8 @@ async def _api_request(self, # Generate auth headers if needed. headers: dict = {"Content-Type": "application/x-www-form-urlencoded"} if is_auth_required: - headers: dict = self._hitbtc_auth.get_headers(method, f"{Constants.REST_URL_AUTH}/{endpoint}", - params) + headers: dict = self._coinzoom_auth.get_headers(method, f"{Constants.REST_URL_AUTH}/{endpoint}", + params) # Build request coro response_coro = shared_client.request(method=method.upper(), url=url, headers=headers, params=qs_params, data=req_form, @@ -339,9 +339,9 @@ async def _api_request(self, return await self._api_request(method=method, endpoint=endpoint, params=params, is_auth_required=is_auth_required, try_count=try_count) else: - raise HitbtcAPIError({"error": parsed_response, "status": http_status}) + raise CoinzoomAPIError({"error": parsed_response, "status": http_status}) if "error" in parsed_response: - raise HitbtcAPIError(parsed_response) + raise CoinzoomAPIError(parsed_response) return parsed_response def get_order_price_quantum(self, trading_pair: str, price: Decimal): @@ -435,7 +435,7 @@ async def _create_order(self, "price": f"{price:f}", "quantity": f"{amount:f}", "clientOrderId": order_id, - # Without strict validate, HitBTC might adjust order prices/sizes. + # Without strict validate, CoinZoom might adjust order prices/sizes. "strictValidate": "true", } if order_type is OrderType.LIMIT_MAKER: @@ -459,7 +459,7 @@ async def _create_order(self, event_cls(self.current_timestamp, order_type, trading_pair, amount, price, order_id)) except asyncio.CancelledError: raise - except HitbtcAPIError as e: + except CoinzoomAPIError as e: error_reason = e.error_payload.get('error', {}).get('message') self.stop_tracking_order(order_id) self.logger().network( @@ -482,7 +482,7 @@ def start_tracking_order(self, """ Starts tracking an order by simply adding it into _in_flight_orders dictionary. """ - self._in_flight_orders[order_id] = HitbtcInFlightOrder( + self._in_flight_orders[order_id] = CoinzoomInFlightOrder( client_order_id=order_id, exchange_order_id=exchange_order_id, trading_pair=trading_pair, @@ -505,7 +505,7 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: """ Executes order cancellation process by first calling cancel-order API. The API result doesn't confirm whether the cancellation is successful, it simply states it receives the request. - :param trading_pair: The market trading pair (Unused during cancel on HitBTC) + :param trading_pair: The market trading pair (Unused during cancel on CoinZoom) :param order_id: The internal order id order.last_state to change to CANCELED """ @@ -523,7 +523,7 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: order_was_cancelled = True except asyncio.CancelledError: raise - except HitbtcAPIError as e: + except CoinzoomAPIError as e: err = e.error_payload.get('error', e.error_payload) self._order_not_found_records[order_id] = self._order_not_found_records.get(order_id, 0) + 1 if err.get('code') == 20002 and \ @@ -596,7 +596,7 @@ async def _update_order_status(self): responses = await safe_gather(*tasks, return_exceptions=True) for response, tracked_order in zip(responses, tracked_orders): client_order_id = tracked_order.client_order_id - if isinstance(response, HitbtcAPIError): + if isinstance(response, CoinzoomAPIError): err = response.error_payload.get('error', response.error_payload) if err.get('code') == 20002: self._order_not_found_records[client_order_id] = \ @@ -822,7 +822,7 @@ async def _iter_user_event_queue(self) -> AsyncIterable[Dict[str, any]]: async def _user_stream_event_listener(self): """ Listens to message in _user_stream_tracker.user_stream queue. The messages are put in by - HitbtcAPIUserStreamDataSource. + CoinzoomAPIUserStreamDataSource. """ async for event_message in self._iter_user_event_queue(): try: diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py b/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py index 54766be2f1..d062d24923 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py @@ -14,7 +14,7 @@ s_decimal_0 = Decimal(0) -class HitbtcInFlightOrder(InFlightOrderBase): +class CoinzoomInFlightOrder(InFlightOrderBase): def __init__(self, client_order_id: str, exchange_order_id: Optional[str], @@ -55,7 +55,7 @@ def from_json(cls, data: Dict[str, Any]) -> InFlightOrderBase: :param data: json data from API :return: formatted InFlightOrder """ - retval = HitbtcInFlightOrder( + retval = CoinzoomInFlightOrder( data["client_order_id"], data["exchange_order_id"], data["trading_pair"], diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book.py b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book.py index 1a3c91a121..96f00de856 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import logging -from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants +from hummingbot.connector.exchange.coinzoom.coinzoom_constants import Constants from sqlalchemy.engine import RowProxy from typing import ( @@ -13,12 +13,12 @@ from hummingbot.core.data_type.order_book_message import ( OrderBookMessage, OrderBookMessageType ) -from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_message import HitbtcOrderBookMessage +from hummingbot.connector.exchange.coinzoom.coinzoom_order_book_message import CoinzoomOrderBookMessage _logger = None -class HitbtcOrderBook(OrderBook): +class CoinzoomOrderBook(OrderBook): @classmethod def logger(cls) -> HummingbotLogger: global _logger @@ -35,13 +35,13 @@ def snapshot_message_from_exchange(cls, Convert json snapshot data into standard OrderBookMessage format :param msg: json snapshot data from live web socket stream :param timestamp: timestamp attached to incoming data - :return: HitbtcOrderBookMessage + :return: CoinzoomOrderBookMessage """ if metadata: msg.update(metadata) - return HitbtcOrderBookMessage( + return CoinzoomOrderBookMessage( message_type=OrderBookMessageType.SNAPSHOT, content=msg, timestamp=timestamp @@ -53,9 +53,9 @@ def snapshot_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = N *used for backtesting Convert a row of snapshot data into standard OrderBookMessage format :param record: a row of snapshot data from the database - :return: HitbtcOrderBookMessage + :return: CoinzoomOrderBookMessage """ - return HitbtcOrderBookMessage( + return CoinzoomOrderBookMessage( message_type=OrderBookMessageType.SNAPSHOT, content=record.json, timestamp=record.timestamp @@ -70,13 +70,13 @@ def diff_message_from_exchange(cls, Convert json diff data into standard OrderBookMessage format :param msg: json diff data from live web socket stream :param timestamp: timestamp attached to incoming data - :return: HitbtcOrderBookMessage + :return: CoinzoomOrderBookMessage """ if metadata: msg.update(metadata) - return HitbtcOrderBookMessage( + return CoinzoomOrderBookMessage( message_type=OrderBookMessageType.DIFF, content=msg, timestamp=timestamp @@ -88,9 +88,9 @@ def diff_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None) *used for backtesting Convert a row of diff data into standard OrderBookMessage format :param record: a row of diff data from the database - :return: HitbtcOrderBookMessage + :return: CoinzoomOrderBookMessage """ - return HitbtcOrderBookMessage( + return CoinzoomOrderBookMessage( message_type=OrderBookMessageType.DIFF, content=record.json, timestamp=record.timestamp @@ -104,7 +104,7 @@ def trade_message_from_exchange(cls, """ Convert a trade data into standard OrderBookMessage format :param record: a trade data from the database - :return: HitbtcOrderBookMessage + :return: CoinzoomOrderBookMessage """ if metadata: @@ -117,7 +117,7 @@ def trade_message_from_exchange(cls, "amount": msg.get("quantity"), }) - return HitbtcOrderBookMessage( + return CoinzoomOrderBookMessage( message_type=OrderBookMessageType.TRADE, content=msg, timestamp=timestamp @@ -129,9 +129,9 @@ def trade_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None *used for backtesting Convert a row of trade data into standard OrderBookMessage format :param record: a row of trade data from the database - :return: HitbtcOrderBookMessage + :return: CoinzoomOrderBookMessage """ - return HitbtcOrderBookMessage( + return CoinzoomOrderBookMessage( message_type=OrderBookMessageType.TRADE, content=record.json, timestamp=record.timestamp diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py index 1f0bc1d631..2eb6529a8c 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py @@ -11,12 +11,12 @@ OrderBookMessage, OrderBookMessageType, ) -from .hitbtc_utils import ( +from .coinzoom_utils import ( convert_from_exchange_trading_pair, ) -class HitbtcOrderBookMessage(OrderBookMessage): +class CoinzoomOrderBookMessage(OrderBookMessage): def __new__( cls, message_type: OrderBookMessageType, @@ -30,7 +30,7 @@ def __new__( raise ValueError("timestamp must not be None when initializing snapshot messages.") timestamp = content["timestamp"] - return super(HitbtcOrderBookMessage, cls).__new__( + return super(CoinzoomOrderBookMessage, cls).__new__( cls, message_type, content, timestamp=timestamp, *args, **kwargs ) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker.py b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker.py index d3161de17e..c81ca9a7bd 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker.py @@ -2,7 +2,7 @@ import asyncio import bisect import logging -from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants +from hummingbot.connector.exchange.coinzoom.coinzoom_constants import Constants import time from collections import defaultdict, deque @@ -10,13 +10,13 @@ from hummingbot.core.data_type.order_book_message import OrderBookMessageType from hummingbot.logger import HummingbotLogger from hummingbot.core.data_type.order_book_tracker import OrderBookTracker -from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_message import HitbtcOrderBookMessage -from hummingbot.connector.exchange.hitbtc.hitbtc_active_order_tracker import HitbtcActiveOrderTracker -from hummingbot.connector.exchange.hitbtc.hitbtc_api_order_book_data_source import HitbtcAPIOrderBookDataSource -from hummingbot.connector.exchange.hitbtc.hitbtc_order_book import HitbtcOrderBook +from hummingbot.connector.exchange.coinzoom.coinzoom_order_book_message import CoinzoomOrderBookMessage +from hummingbot.connector.exchange.coinzoom.coinzoom_active_order_tracker import CoinzoomActiveOrderTracker +from hummingbot.connector.exchange.coinzoom.coinzoom_api_order_book_data_source import CoinzoomAPIOrderBookDataSource +from hummingbot.connector.exchange.coinzoom.coinzoom_order_book import CoinzoomOrderBook -class HitbtcOrderBookTracker(OrderBookTracker): +class CoinzoomOrderBookTracker(OrderBookTracker): _logger: Optional[HummingbotLogger] = None @classmethod @@ -26,7 +26,7 @@ def logger(cls) -> HummingbotLogger: return cls._logger def __init__(self, trading_pairs: Optional[List[str]] = None,): - super().__init__(HitbtcAPIOrderBookDataSource(trading_pairs), trading_pairs) + super().__init__(CoinzoomAPIOrderBookDataSource(trading_pairs), trading_pairs) self._ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() self._order_book_snapshot_stream: asyncio.Queue = asyncio.Queue() @@ -34,10 +34,10 @@ def __init__(self, trading_pairs: Optional[List[str]] = None,): self._order_book_trade_stream: asyncio.Queue = asyncio.Queue() self._process_msg_deque_task: Optional[asyncio.Task] = None self._past_diffs_windows: Dict[str, Deque] = {} - self._order_books: Dict[str, HitbtcOrderBook] = {} - self._saved_message_queues: Dict[str, Deque[HitbtcOrderBookMessage]] = \ + self._order_books: Dict[str, CoinzoomOrderBook] = {} + self._saved_message_queues: Dict[str, Deque[CoinzoomOrderBookMessage]] = \ defaultdict(lambda: deque(maxlen=1000)) - self._active_order_trackers: Dict[str, HitbtcActiveOrderTracker] = defaultdict(HitbtcActiveOrderTracker) + self._active_order_trackers: Dict[str, CoinzoomActiveOrderTracker] = defaultdict(CoinzoomActiveOrderTracker) self._order_book_stream_listener_task: Optional[asyncio.Task] = None self._order_book_trade_listener_task: Optional[asyncio.Task] = None @@ -52,20 +52,20 @@ async def _track_single_book(self, trading_pair: str): """ Update an order book with changes from the latest batch of received messages """ - past_diffs_window: Deque[HitbtcOrderBookMessage] = deque() + past_diffs_window: Deque[CoinzoomOrderBookMessage] = deque() self._past_diffs_windows[trading_pair] = past_diffs_window message_queue: asyncio.Queue = self._tracking_message_queues[trading_pair] - order_book: HitbtcOrderBook = self._order_books[trading_pair] - active_order_tracker: HitbtcActiveOrderTracker = self._active_order_trackers[trading_pair] + order_book: CoinzoomOrderBook = self._order_books[trading_pair] + active_order_tracker: CoinzoomActiveOrderTracker = self._active_order_trackers[trading_pair] last_message_timestamp: float = time.time() diff_messages_accepted: int = 0 while True: try: - message: HitbtcOrderBookMessage = None - saved_messages: Deque[HitbtcOrderBookMessage] = self._saved_message_queues[trading_pair] + message: CoinzoomOrderBookMessage = None + saved_messages: Deque[CoinzoomOrderBookMessage] = self._saved_message_queues[trading_pair] # Process saved messages first if there are any if len(saved_messages) > 0: message = saved_messages.popleft() @@ -87,7 +87,7 @@ async def _track_single_book(self, trading_pair: str): diff_messages_accepted = 0 last_message_timestamp = now elif message.type is OrderBookMessageType.SNAPSHOT: - past_diffs: List[HitbtcOrderBookMessage] = list(past_diffs_window) + past_diffs: List[CoinzoomOrderBookMessage] = list(past_diffs_window) # only replay diffs later than snapshot, first update active order with snapshot then replay diffs replay_position = bisect.bisect_right(past_diffs, message) replay_diffs = past_diffs[replay_position:] diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker_entry.py b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker_entry.py index 5edfbadec0..94feda5275 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker_entry.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_tracker_entry.py @@ -1,21 +1,21 @@ from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.order_book_tracker_entry import OrderBookTrackerEntry -from hummingbot.connector.exchange.hitbtc.hitbtc_active_order_tracker import HitbtcActiveOrderTracker +from hummingbot.connector.exchange.coinzoom.coinzoom_active_order_tracker import CoinzoomActiveOrderTracker -class HitbtcOrderBookTrackerEntry(OrderBookTrackerEntry): +class CoinzoomOrderBookTrackerEntry(OrderBookTrackerEntry): def __init__( - self, trading_pair: str, timestamp: float, order_book: OrderBook, active_order_tracker: HitbtcActiveOrderTracker + self, trading_pair: str, timestamp: float, order_book: OrderBook, active_order_tracker: CoinzoomActiveOrderTracker ): self._active_order_tracker = active_order_tracker - super(HitbtcOrderBookTrackerEntry, self).__init__(trading_pair, timestamp, order_book) + super(CoinzoomOrderBookTrackerEntry, self).__init__(trading_pair, timestamp, order_book) def __repr__(self) -> str: return ( - f"HitbtcOrderBookTrackerEntry(trading_pair='{self._trading_pair}', timestamp='{self._timestamp}', " + f"CoinzoomOrderBookTrackerEntry(trading_pair='{self._trading_pair}', timestamp='{self._timestamp}', " f"order_book='{self._order_book}')" ) @property - def active_order_tracker(self) -> HitbtcActiveOrderTracker: + def active_order_tracker(self) -> CoinzoomActiveOrderTracker: return self._active_order_tracker diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_user_stream_tracker.py b/hummingbot/connector/exchange/coinzoom/coinzoom_user_stream_tracker.py index 7b04002ccd..79c7584d70 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_user_stream_tracker.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_user_stream_tracker.py @@ -15,13 +15,13 @@ safe_ensure_future, safe_gather, ) -from hummingbot.connector.exchange.hitbtc.hitbtc_api_user_stream_data_source import \ - HitbtcAPIUserStreamDataSource -from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth -from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants +from hummingbot.connector.exchange.coinzoom.coinzoom_api_user_stream_data_source import \ + CoinzoomAPIUserStreamDataSource +from hummingbot.connector.exchange.coinzoom.coinzoom_auth import CoinzoomAuth +from hummingbot.connector.exchange.coinzoom.coinzoom_constants import Constants -class HitbtcUserStreamTracker(UserStreamTracker): +class CoinzoomUserStreamTracker(UserStreamTracker): _cbpust_logger: Optional[HummingbotLogger] = None @classmethod @@ -31,10 +31,10 @@ def logger(cls) -> HummingbotLogger: return cls._bust_logger def __init__(self, - hitbtc_auth: Optional[HitbtcAuth] = None, + coinzoom_auth: Optional[CoinzoomAuth] = None, trading_pairs: Optional[List[str]] = []): super().__init__() - self._hitbtc_auth: HitbtcAuth = hitbtc_auth + self._coinzoom_auth: CoinzoomAuth = coinzoom_auth self._trading_pairs: List[str] = trading_pairs self._ev_loop: asyncio.events.AbstractEventLoop = asyncio.get_event_loop() self._data_source: Optional[UserStreamTrackerDataSource] = None @@ -48,8 +48,8 @@ def data_source(self) -> UserStreamTrackerDataSource: :return: OrderBookTrackerDataSource """ if not self._data_source: - self._data_source = HitbtcAPIUserStreamDataSource( - hitbtc_auth=self._hitbtc_auth, + self._data_source = CoinzoomAPIUserStreamDataSource( + coinzoom_auth=self._coinzoom_auth, trading_pairs=self._trading_pairs ) return self._data_source diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py index c549ce8b72..f593b69b99 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py @@ -13,7 +13,7 @@ from hummingbot.core.utils.tracking_nonce import get_tracking_nonce from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.config_methods import using_exchange -from .hitbtc_constants import Constants +from .coinzoom_constants import Constants TRADING_PAIR_SPLITTER = re.compile(Constants.TRADING_PAIR_SPLITTER) @@ -25,7 +25,7 @@ DEFAULT_FEES = [0.1, 0.25] -class HitbtcAPIError(IOError): +class CoinzoomAPIError(IOError): def __init__(self, error_payload: Dict[str, Any]): super().__init__(str(error_payload)) self.error_payload = error_payload @@ -61,13 +61,13 @@ def convert_from_exchange_trading_pair(ex_trading_pair: str) -> Optional[str]: regex_match = split_trading_pair(ex_trading_pair) if regex_match is None: return None - # HitBTC uses uppercase (BTCUSDT) + # CoinZoom uses uppercase (BTCUSDT) base_asset, quote_asset = split_trading_pair(ex_trading_pair) return f"{base_asset.upper()}-{quote_asset.upper()}" def convert_to_exchange_trading_pair(hb_trading_pair: str) -> str: - # HitBTC uses uppercase (BTCUSDT) + # CoinZoom uses uppercase (BTCUSDT) return hb_trading_pair.replace("-", "").upper() @@ -136,21 +136,21 @@ async def api_call_with_retries(method, return await api_call_with_retries(method=method, endpoint=endpoint, params=params, shared_client=shared_client, try_count=try_count) else: - raise HitbtcAPIError({"error": parsed_response, "status": http_status}) + raise CoinzoomAPIError({"error": parsed_response, "status": http_status}) return parsed_response KEYS = { - "hitbtc_api_key": - ConfigVar(key="hitbtc_api_key", + "coinzoom_api_key": + ConfigVar(key="coinzoom_api_key", prompt=f"Enter your {Constants.EXCHANGE_NAME} API key >>> ", - required_if=using_exchange("hitbtc"), + required_if=using_exchange("coinzoom"), is_secure=True, is_connect_key=True), - "hitbtc_secret_key": - ConfigVar(key="hitbtc_secret_key", + "coinzoom_secret_key": + ConfigVar(key="coinzoom_secret_key", prompt=f"Enter your {Constants.EXCHANGE_NAME} secret key >>> ", - required_if=using_exchange("hitbtc"), + required_if=using_exchange("coinzoom"), is_secure=True, is_connect_key=True), } diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py b/hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py index da65b869a2..a9021eee62 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py @@ -4,7 +4,7 @@ import logging import websockets import json -from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants +from hummingbot.connector.exchange.coinzoom.coinzoom_constants import Constants from typing import ( @@ -15,17 +15,17 @@ ) from websockets.exceptions import ConnectionClosed from hummingbot.logger import HummingbotLogger -from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth -from hummingbot.connector.exchange.hitbtc.hitbtc_utils import ( +from hummingbot.connector.exchange.coinzoom.coinzoom_auth import CoinzoomAuth +from hummingbot.connector.exchange.coinzoom.coinzoom_utils import ( RequestId, - HitbtcAPIError, + CoinzoomAPIError, ) # reusable websocket class # ToDo: We should eventually remove this class, and instantiate web socket connection normally (see Binance for example) -class HitbtcWebsocket(RequestId): +class CoinzoomWebsocket(RequestId): _logger: Optional[HummingbotLogger] = None @classmethod @@ -35,8 +35,8 @@ def logger(cls) -> HummingbotLogger: return cls._logger def __init__(self, - auth: Optional[HitbtcAuth] = None): - self._auth: Optional[HitbtcAuth] = auth + auth: Optional[CoinzoomAuth] = None): + self._auth: Optional[CoinzoomAuth] = auth self._isPrivate = True if self._auth is not None else False self._WS_URL = Constants.WS_PRIVATE_URL if self._isPrivate else Constants.WS_PUBLIC_URL self._client: Optional[websockets.WebSocketClientProtocol] = None @@ -54,7 +54,7 @@ async def connect(self): json_msg = json.loads(raw_msg_str) if json_msg.get("result") is not True: err_msg = json_msg.get('error', {}).get('message') - raise HitbtcAPIError({"error": f"Failed to authenticate to websocket - {err_msg}."}) + raise CoinzoomAPIError({"error": f"Failed to authenticate to websocket - {err_msg}."}) return self._client @@ -73,7 +73,7 @@ async def _messages(self) -> AsyncIterable[Any]: raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=Constants.MESSAGE_TIMEOUT) try: msg = json.loads(raw_msg_str) - # HitBTC doesn't support ping or heartbeat messages. + # CoinZoom doesn't support ping or heartbeat messages. # Can handle them here if that changes - use `safe_ensure_future`. yield msg except ValueError: diff --git a/test/connector/exchange/coinzoom/test_coinzoom_auth.py b/test/connector/exchange/coinzoom/test_coinzoom_auth.py index 6cc71c27e9..9accc46e38 100644 --- a/test/connector/exchange/coinzoom/test_coinzoom_auth.py +++ b/test/connector/exchange/coinzoom/test_coinzoom_auth.py @@ -7,10 +7,10 @@ import logging from os.path import join, realpath from typing import Dict, Any -from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth -from hummingbot.connector.exchange.hitbtc.hitbtc_websocket import HitbtcWebsocket +from hummingbot.connector.exchange.coinzoom.coinzoom_auth import CoinzoomAuth +from hummingbot.connector.exchange.coinzoom.coinzoom_websocket import CoinzoomWebsocket from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL -from hummingbot.connector.exchange.hitbtc.hitbtc_constants import Constants +from hummingbot.connector.exchange.coinzoom.coinzoom_constants import Constants sys.path.insert(0, realpath(join(__file__, "../../../../../"))) logging.basicConfig(level=METRICS_LOG_LEVEL) @@ -20,9 +20,9 @@ class TestAuth(unittest.TestCase): @classmethod def setUpClass(cls): cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() - api_key = conf.hitbtc_api_key - secret_key = conf.hitbtc_secret_key - cls.auth = HitbtcAuth(api_key, secret_key) + api_key = conf.coinzoom_api_key + secret_key = conf.coinzoom_secret_key + cls.auth = CoinzoomAuth(api_key, secret_key) async def rest_auth(self) -> Dict[Any, Any]: endpoint = Constants.ENDPOINT['USER_BALANCES'] @@ -31,7 +31,7 @@ async def rest_auth(self) -> Dict[Any, Any]: return await response.json() async def ws_auth(self) -> Dict[Any, Any]: - ws = HitbtcWebsocket(self.auth) + ws = CoinzoomWebsocket(self.auth) await ws.connect() await ws.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"], None, {}) async for response in ws.on_message(): diff --git a/test/connector/exchange/coinzoom/test_coinzoom_exchange.py b/test/connector/exchange/coinzoom/test_coinzoom_exchange.py index 0456f5a8a9..6a968ed97a 100644 --- a/test/connector/exchange/coinzoom/test_coinzoom_exchange.py +++ b/test/connector/exchange/coinzoom/test_coinzoom_exchange.py @@ -33,15 +33,15 @@ from hummingbot.model.order import Order from hummingbot.model.trade_fill import TradeFill from hummingbot.connector.markets_recorder import MarketsRecorder -from hummingbot.connector.exchange.hitbtc.hitbtc_exchange import HitbtcExchange +from hummingbot.connector.exchange.coinzoom.coinzoom_exchange import CoinzoomExchange logging.basicConfig(level=METRICS_LOG_LEVEL) -API_KEY = conf.hitbtc_api_key -API_SECRET = conf.hitbtc_secret_key +API_KEY = conf.coinzoom_api_key +API_SECRET = conf.coinzoom_secret_key -class HitbtcExchangeUnitTest(unittest.TestCase): +class CoinzoomExchangeUnitTest(unittest.TestCase): events: List[MarketEvent] = [ MarketEvent.BuyOrderCompleted, MarketEvent.SellOrderCompleted, @@ -52,7 +52,7 @@ class HitbtcExchangeUnitTest(unittest.TestCase): MarketEvent.OrderCancelled, MarketEvent.OrderFailure ] - connector: HitbtcExchange + connector: CoinzoomExchange event_logger: EventLogger trading_pair = "BTC-USD" base_token, quote_token = trading_pair.split("-") @@ -65,13 +65,13 @@ def setUpClass(cls): cls.ev_loop = asyncio.get_event_loop() cls.clock: Clock = Clock(ClockMode.REALTIME) - cls.connector: HitbtcExchange = HitbtcExchange( - hitbtc_api_key=API_KEY, - hitbtc_secret_key=API_SECRET, + cls.connector: CoinzoomExchange = CoinzoomExchange( + coinzoom_api_key=API_KEY, + coinzoom_secret_key=API_SECRET, trading_pairs=[cls.trading_pair], trading_required=True ) - print("Initializing Hitbtc market... this will take about a minute.") + print("Initializing Coinzoom market... this will take about a minute.") cls.clock.add_iterator(cls.connector) cls.stack: contextlib.ExitStack = contextlib.ExitStack() cls._clock = cls.stack.enter_context(cls.clock) @@ -351,7 +351,7 @@ def test_orders_saving_and_restoration(self): self.connector.remove_listener(event_tag, self.event_logger) # Clear the event loop self.event_logger.clear() - new_connector = HitbtcExchange(API_KEY, API_SECRET, [self.trading_pair], True) + new_connector = CoinzoomExchange(API_KEY, API_SECRET, [self.trading_pair], True) for event_tag in self.events: new_connector.add_listener(event_tag, self.event_logger) recorder.stop() diff --git a/test/connector/exchange/coinzoom/test_coinzoom_order_book_tracker.py b/test/connector/exchange/coinzoom/test_coinzoom_order_book_tracker.py index ae3778e7c9..8aaed4c071 100755 --- a/test/connector/exchange/coinzoom/test_coinzoom_order_book_tracker.py +++ b/test/connector/exchange/coinzoom/test_coinzoom_order_book_tracker.py @@ -9,8 +9,8 @@ from typing import Dict, Optional, List from hummingbot.core.event.event_logger import EventLogger from hummingbot.core.event.events import OrderBookEvent, OrderBookTradeEvent, TradeType -from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_tracker import HitbtcOrderBookTracker -from hummingbot.connector.exchange.hitbtc.hitbtc_api_order_book_data_source import HitbtcAPIOrderBookDataSource +from hummingbot.connector.exchange.coinzoom.coinzoom_order_book_tracker import CoinzoomOrderBookTracker +from hummingbot.connector.exchange.coinzoom.coinzoom_api_order_book_data_source import CoinzoomAPIOrderBookDataSource from hummingbot.core.data_type.order_book import OrderBook from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL @@ -19,8 +19,8 @@ logging.basicConfig(level=METRICS_LOG_LEVEL) -class HitbtcOrderBookTrackerUnitTest(unittest.TestCase): - order_book_tracker: Optional[HitbtcOrderBookTracker] = None +class CoinzoomOrderBookTrackerUnitTest(unittest.TestCase): + order_book_tracker: Optional[CoinzoomOrderBookTracker] = None events: List[OrderBookEvent] = [ OrderBookEvent.TradeEvent ] @@ -32,7 +32,7 @@ class HitbtcOrderBookTrackerUnitTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() - cls.order_book_tracker: HitbtcOrderBookTracker = HitbtcOrderBookTracker(cls.trading_pairs) + cls.order_book_tracker: CoinzoomOrderBookTracker = CoinzoomOrderBookTracker(cls.trading_pairs) cls.order_book_tracker.start() cls.ev_loop.run_until_complete(cls.wait_til_tracker_ready()) @@ -96,7 +96,7 @@ def test_tracker_integrity(self): def test_api_get_last_traded_prices(self): prices = self.ev_loop.run_until_complete( - HitbtcAPIOrderBookDataSource.get_last_traded_prices(["BTC-USD", "LTC-BTC"])) + CoinzoomAPIOrderBookDataSource.get_last_traded_prices(["BTC-USD", "LTC-BTC"])) for key, value in prices.items(): print(f"{key} last_trade_price: {value}") self.assertGreater(prices["BTC-USD"], 1000) diff --git a/test/connector/exchange/coinzoom/test_coinzoom_user_stream_tracker.py b/test/connector/exchange/coinzoom/test_coinzoom_user_stream_tracker.py index 5c82f2372b..8a6337eaa5 100644 --- a/test/connector/exchange/coinzoom/test_coinzoom_user_stream_tracker.py +++ b/test/connector/exchange/coinzoom/test_coinzoom_user_stream_tracker.py @@ -7,8 +7,8 @@ import conf from os.path import join, realpath -from hummingbot.connector.exchange.hitbtc.hitbtc_user_stream_tracker import HitbtcUserStreamTracker -from hummingbot.connector.exchange.hitbtc.hitbtc_auth import HitbtcAuth +from hummingbot.connector.exchange.coinzoom.coinzoom_user_stream_tracker import CoinzoomUserStreamTracker +from hummingbot.connector.exchange.coinzoom.coinzoom_auth import CoinzoomAuth from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL @@ -17,16 +17,16 @@ logging.basicConfig(level=METRICS_LOG_LEVEL) -class HitbtcUserStreamTrackerUnitTest(unittest.TestCase): - api_key = conf.hitbtc_api_key - api_secret = conf.hitbtc_secret_key +class CoinzoomUserStreamTrackerUnitTest(unittest.TestCase): + api_key = conf.coinzoom_api_key + api_secret = conf.coinzoom_secret_key @classmethod def setUpClass(cls): cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() cls.trading_pairs = ["BTC-USD"] - cls.user_stream_tracker: HitbtcUserStreamTracker = HitbtcUserStreamTracker( - hitbtc_auth=HitbtcAuth(cls.api_key, cls.api_secret), + cls.user_stream_tracker: CoinzoomUserStreamTracker = CoinzoomUserStreamTracker( + coinzoom_auth=CoinzoomAuth(cls.api_key, cls.api_secret), trading_pairs=cls.trading_pairs) cls.user_stream_tracker_task: asyncio.Task = safe_ensure_future(cls.user_stream_tracker.start()) From 5a8657ae35a07866d505ebd7bb0c529ba426f381 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Sat, 20 Mar 2021 13:55:11 +0000 Subject: [PATCH 03/23] CoinZoom: Start work on Websockets and Auth --- conf/__init__.py | 1 + .../exchange/coinzoom/coinzoom_auth.py | 61 +++----------- .../exchange/coinzoom/coinzoom_constants.py | 16 ++-- .../exchange/coinzoom/coinzoom_exchange.py | 9 +- .../exchange/coinzoom/coinzoom_utils.py | 29 ++----- .../exchange/coinzoom/coinzoom_websocket.py | 82 ++++++++++--------- hummingbot/templates/conf_global_TEMPLATE.yml | 1 + .../exchange/coinzoom/test_coinzoom_auth.py | 27 +++--- .../coinzoom/test_coinzoom_exchange.py | 4 +- .../test_coinzoom_user_stream_tracker.py | 3 +- 10 files changed, 96 insertions(+), 137 deletions(-) diff --git a/conf/__init__.py b/conf/__init__.py index f096f7f933..69e87838fa 100644 --- a/conf/__init__.py +++ b/conf/__init__.py @@ -111,6 +111,7 @@ # CoinZoom Test coinzoom_api_key = os.getenv("COINZOOM_API_KEY") coinzoom_secret_key = os.getenv("COINZOOM_SECRET_KEY") +coinzoom_username = os.getenv("COINZOOM_USERNAME") # Wallet Tests test_erc20_token_address = os.getenv("TEST_ERC20_TOKEN_ADDRESS") diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_auth.py b/hummingbot/connector/exchange/coinzoom/coinzoom_auth.py index 550b1dd6e9..9379f3716b 100755 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_auth.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_auth.py @@ -1,7 +1,3 @@ -import hmac -import hashlib -import time -from base64 import b64encode from typing import Dict, Any @@ -10,63 +6,26 @@ class CoinzoomAuth(): Auth class required by CoinZoom API Learn more at https://exchange-docs.crypto.com/#digital-signature """ - def __init__(self, api_key: str, secret_key: str): + def __init__(self, api_key: str, secret_key: str, username: str): self.api_key = api_key self.secret_key = secret_key + self.username = username - def generate_payload( - self, - method: str, - url: str, - params: Dict[str, Any] = None, - ): - """ - Generates authentication payload and returns it. - :return: A base64 encoded payload for the authentication header. - """ - # Nonce is standard EPOCH timestamp only accurate to 1s - nonce = str(int(time.time())) - body = "" - # Need to build the full URL with query string for HS256 sig - if params is not None and len(params) > 0: - query_string = "&".join([f"{k}={v}" for k, v in params.items()]) - if method == "GET": - url = f"{url}?{query_string}" - else: - body = query_string - # Concat payload - payload = f"{method}{nonce}{url}{body}" - # Create HS256 sig - sig = hmac.new(self.secret_key.encode(), payload.encode(), hashlib.sha256).hexdigest() - # Base64 encode it with public key and nonce - return b64encode(f"{self.api_key}:{nonce}:{sig}".encode()).decode().strip() - - def generate_auth_dict_ws(self, - nonce: int): - """ - Generates an authentication params for CoinZoom websockets login - :return: a dictionary of auth params - """ + def get_ws_params(self) -> Dict[str, str]: return { - "algo": "HS256", - "pKey": str(self.api_key), - "nonce": str(nonce), - "signature": hmac.new(self.secret_key.encode('utf-8'), - str(nonce).encode('utf-8'), - hashlib.sha256).hexdigest() + "apiKey": str(self.api_key), + "secretKey": str(self.secret_key), } - def get_headers(self, - method, - url, - params) -> Dict[str, Any]: + def get_headers(self) -> Dict[str, Any]: """ Generates authentication headers required by CoinZoom :return: a dictionary of auth headers """ - payload = self.generate_payload(method, url, params) headers = { - "Authorization": f"HS256 {payload}", - "Content-Type": "application/x-www-form-urlencoded", + "Content-Type": "application/json", + "Coinzoom-Api-Key": str(self.api_key), + "Coinzoom-Api-Secret": str(self.secret_key), + "User-Agent": f"hummingbot ZoomMe: {self.username}" } return headers diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py index 43d69678b4..f8cfcb2f31 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py @@ -1,10 +1,9 @@ # A single source of truth for constant variables related to the exchange class Constants: EXCHANGE_NAME = "coinzoom" - REST_URL = "https://api.coinzoom.com/api/2" - REST_URL_AUTH = "/api/2" - WS_PRIVATE_URL = "wss://api.coinzoom.com/api/2/ws/trading" - WS_PUBLIC_URL = "wss://api.coinzoom.com/api/2/ws/public" + REST_URL = "https://api.stage.coinzoom.com/api/v1/public" + WS_PRIVATE_URL = "wss://api.stage.coinzoom.com/api/v1/public/market/data/stream" + WS_PUBLIC_URL = "wss://api.stage.coinzoom.com/api/v1/public/market/data/stream" HBOT_BROKER_ID = "refzzz48" @@ -18,13 +17,13 @@ class Constants: "ORDER_DELETE": "order/{id}", "ORDER_STATUS": "order/{id}", "USER_ORDERS": "order", - "USER_BALANCES": "trading/balance", + "USER_BALANCES": "ledger/list", } WS_SUB = { - "TRADES": "Trades", + "TRADES": "TradeSummaryRequest", "ORDERS": "Orderbook", - "USER_ORDERS_TRADES": "Reports", + "USER_ORDERS_TRADES": ["OrderUpdateRequest"], } @@ -52,6 +51,3 @@ class Constants: UPDATE_ORDER_STATUS_INTERVAL = 60.0 # 10 minute interval to update trading rules, these would likely never change whilst running. INTERVAL_TRADING_RULES = 600 - - # Trading pair splitter regex - TRADING_PAIR_SPLITTER = r"^(\w+)(BTC|BCH|DAI|DDRST|EOSDT|EOS|ETH|EURS|IDRT|PAX|BUSD|GUSD|TUSD|USDC|USDT|USD)$" diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py index d9eaf36f71..552dafe20f 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py @@ -72,19 +72,21 @@ def logger(cls) -> HummingbotLogger: def __init__(self, coinzoom_api_key: str, coinzoom_secret_key: str, + coinzoom_username: str, trading_pairs: Optional[List[str]] = None, trading_required: bool = True ): """ :param coinzoom_api_key: The API key to connect to private CoinZoom APIs. :param coinzoom_secret_key: The API secret. + :param coinzoom_username: The ZoomMe Username. :param trading_pairs: The market trading pairs which to track order book data. :param trading_required: Whether actual trading is needed. """ super().__init__() self._trading_required = trading_required self._trading_pairs = trading_pairs - self._coinzoom_auth = CoinzoomAuth(coinzoom_api_key, coinzoom_secret_key) + self._coinzoom_auth = CoinzoomAuth(coinzoom_api_key, coinzoom_secret_key, coinzoom_username) self._order_book_tracker = CoinzoomOrderBookTracker(trading_pairs=trading_pairs) self._user_stream_tracker = CoinzoomUserStreamTracker(self._coinzoom_auth, trading_pairs) self._ev_loop = asyncio.get_event_loop() @@ -741,8 +743,9 @@ def _process_balance_message(self, balance_update): remote_asset_names = set() for account in balance_update: asset_name = account["currency"] - self._account_available_balances[asset_name] = Decimal(str(account["available"])) - self._account_balances[asset_name] = Decimal(str(account["reserved"])) + Decimal(str(account["available"])) + total_bal = Decimal(str(account["totalBalance"])) + self._account_available_balances[asset_name] = total_bal + Decimal(str(account["reservedBalance"])) + self._account_balances[asset_name] = total_bal remote_asset_names.add(asset_name) asset_names_to_remove = local_asset_names.difference(remote_asset_names) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py index f593b69b99..c9d285fca0 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py @@ -1,13 +1,11 @@ import aiohttp import asyncio import random -import re from dateutil.parser import parse as dateparse from typing import ( Any, Dict, Optional, - Tuple, ) from hummingbot.core.utils.tracking_nonce import get_tracking_nonce @@ -16,8 +14,6 @@ from .coinzoom_constants import Constants -TRADING_PAIR_SPLITTER = re.compile(Constants.TRADING_PAIR_SPLITTER) - CENTRALIZED = True EXAMPLE_PAIR = "BTC-USD" @@ -48,27 +44,14 @@ def generate_request_id(cls) -> int: return get_tracking_nonce() -def split_trading_pair(trading_pair: str) -> Optional[Tuple[str, str]]: - try: - m = TRADING_PAIR_SPLITTER.match(trading_pair) - return m.group(1), m.group(2) - # Exceptions are now logged as warnings in trading pair fetcher - except Exception: - return None - - def convert_from_exchange_trading_pair(ex_trading_pair: str) -> Optional[str]: - regex_match = split_trading_pair(ex_trading_pair) - if regex_match is None: - return None - # CoinZoom uses uppercase (BTCUSDT) - base_asset, quote_asset = split_trading_pair(ex_trading_pair) - return f"{base_asset.upper()}-{quote_asset.upper()}" + # CoinZoom uses uppercase (BTC/USDT) + return ex_trading_pair.replace("/", "-") def convert_to_exchange_trading_pair(hb_trading_pair: str) -> str: # CoinZoom uses uppercase (BTCUSDT) - return hb_trading_pair.replace("-", "").upper() + return hb_trading_pair.replace("-", "/").upper() def get_new_client_order_id(is_buy: bool, trading_pair: str) -> str: @@ -153,4 +136,10 @@ async def api_call_with_retries(method, required_if=using_exchange("coinzoom"), is_secure=True, is_connect_key=True), + "coinzoom_username": + ConfigVar(key="coinzoom_username", + prompt=f"Enter your {Constants.EXCHANGE_NAME} ZoomMe username >>> ", + required_if=using_exchange("coinzoom"), + is_secure=True, + is_connect_key=True), } diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py b/hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py index a9021eee62..4764196258 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py @@ -1,6 +1,5 @@ #!/usr/bin/env python import asyncio -import copy import logging import websockets import json @@ -11,21 +10,18 @@ Any, AsyncIterable, Dict, + List, Optional, ) from websockets.exceptions import ConnectionClosed from hummingbot.logger import HummingbotLogger from hummingbot.connector.exchange.coinzoom.coinzoom_auth import CoinzoomAuth -from hummingbot.connector.exchange.coinzoom.coinzoom_utils import ( - RequestId, - CoinzoomAPIError, -) # reusable websocket class # ToDo: We should eventually remove this class, and instantiate web socket connection normally (see Binance for example) -class CoinzoomWebsocket(RequestId): +class CoinzoomWebsocket(): _logger: Optional[HummingbotLogger] = None @classmethod @@ -40,21 +36,18 @@ def __init__(self, self._isPrivate = True if self._auth is not None else False self._WS_URL = Constants.WS_PRIVATE_URL if self._isPrivate else Constants.WS_PUBLIC_URL self._client: Optional[websockets.WebSocketClientProtocol] = None + self._is_subscribed = False + + @property + def is_subscribed(self): + return self._is_subscribed # connect to exchange async def connect(self): - self._client = await websockets.connect(self._WS_URL) - # if auth class was passed into websocket class # we need to emit authenticated requests - if self._isPrivate: - auth_params = self._auth.generate_auth_dict_ws(self.generate_request_id()) - await self._emit("login", auth_params, no_id=True) - raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=Constants.MESSAGE_TIMEOUT) - json_msg = json.loads(raw_msg_str) - if json_msg.get("result") is not True: - err_msg = json_msg.get('error', {}).get('message') - raise CoinzoomAPIError({"error": f"Failed to authenticate to websocket - {err_msg}."}) + extra_headers = self._auth.get_headers() if self._isPrivate else None + self._client = await websockets.connect(self._WS_URL, extra_headers=extra_headers) return self._client @@ -73,9 +66,20 @@ async def _messages(self) -> AsyncIterable[Any]: raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=Constants.MESSAGE_TIMEOUT) try: msg = json.loads(raw_msg_str) + # DEBUG PRINTOUT + print(f"WS Msg: {msg}") + # DEBUG PRINTOUT # CoinZoom doesn't support ping or heartbeat messages. # Can handle them here if that changes - use `safe_ensure_future`. - yield msg + result: List[str] = list([d['result'] for k, d in msg.items() if d.get('result') is not None]) + if len(result): + if result[0] == 'subscribed': + self._is_subscribed = True + elif result[0] == 'unsubscribed': + self._is_subscribed = False + yield None + else: + yield msg except ValueError: continue except asyncio.TimeoutError: @@ -89,40 +93,44 @@ async def _messages(self) -> AsyncIterable[Any]: await self.disconnect() # emit messages - async def _emit(self, method: str, data: Optional[Dict[str, Any]] = {}, no_id: bool = False) -> int: - id = self.generate_request_id() - + async def _emit(self, method: str, action: str, data: Optional[Dict[str, Any]] = {}) -> int: payload = { - "id": id, - "method": method, - "params": copy.deepcopy(data), + method: { + "action": action, + **data + } } + # payload = {**payload, **data} + ws_data = json.dumps(payload) + # DEBUG PRINTOUT + print(f"WS Req: {ws_data}") + # DEBUG PRINTOUT await self._client.send(json.dumps(payload)) return id # request via websocket - async def request(self, method: str, data: Optional[Dict[str, Any]] = {}) -> int: - return await self._emit(method, data) + async def request(self, method: str, action: str, data: Optional[Dict[str, Any]] = {}) -> int: + return await self._emit(method, action, data) # subscribe to a method async def subscribe(self, - channel: str, - trading_pair: Optional[str] = None, - params: Optional[Dict[str, Any]] = {}) -> int: - if trading_pair is not None: - params['symbol'] = trading_pair - return await self.request(f"subscribe{channel}", params) + streams: Optional[Dict[str, Any]] = {}) -> int: + for stream, stream_dict in streams.items(): + if self._isPrivate: + stream_dict = {**stream_dict, **self._auth.get_ws_params()} + await self.request(stream, "subscribe", stream_dict) + return True # unsubscribe to a method async def unsubscribe(self, - channel: str, - trading_pair: Optional[str] = None, - params: Optional[Dict[str, Any]] = {}) -> int: - if trading_pair is not None: - params['symbol'] = trading_pair - return await self.request(f"unsubscribe{channel}", params) + streams: Optional[Dict[str, Any]] = {}) -> int: + for stream, stream_dict in streams.items(): + if self._isPrivate: + stream_dict = {**stream_dict, **self._auth.get_ws_params()} + await self.request(stream, "unsubscribe", stream_dict) + return True # listen to messages by method async def on_message(self) -> AsyncIterable[Any]: diff --git a/hummingbot/templates/conf_global_TEMPLATE.yml b/hummingbot/templates/conf_global_TEMPLATE.yml index 4424f82650..4fd7853924 100644 --- a/hummingbot/templates/conf_global_TEMPLATE.yml +++ b/hummingbot/templates/conf_global_TEMPLATE.yml @@ -36,6 +36,7 @@ coinbase_pro_passphrase: null coinzoom_api_key: null coinzoom_secret_key: null +coinzoom_username: null dydx_eth_private_key: null dydx_node_address: null diff --git a/test/connector/exchange/coinzoom/test_coinzoom_auth.py b/test/connector/exchange/coinzoom/test_coinzoom_auth.py index 9accc46e38..bfeeb86cbd 100644 --- a/test/connector/exchange/coinzoom/test_coinzoom_auth.py +++ b/test/connector/exchange/coinzoom/test_coinzoom_auth.py @@ -5,6 +5,7 @@ import aiohttp import conf import logging +from async_timeout import timeout from os.path import join, realpath from typing import Dict, Any from hummingbot.connector.exchange.coinzoom.coinzoom_auth import CoinzoomAuth @@ -22,20 +23,25 @@ def setUpClass(cls): cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() api_key = conf.coinzoom_api_key secret_key = conf.coinzoom_secret_key - cls.auth = CoinzoomAuth(api_key, secret_key) + api_username = conf.coinzoom_username + cls.auth = CoinzoomAuth(api_key, secret_key, api_username) async def rest_auth(self) -> Dict[Any, Any]: endpoint = Constants.ENDPOINT['USER_BALANCES'] - headers = self.auth.get_headers("GET", f"{Constants.REST_URL_AUTH}/{endpoint}", None) + headers = self.auth.get_headers() response = await aiohttp.ClientSession().get(f"{Constants.REST_URL}/{endpoint}", headers=headers) return await response.json() async def ws_auth(self) -> Dict[Any, Any]: ws = CoinzoomWebsocket(self.auth) await ws.connect() - await ws.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"], None, {}) - async for response in ws.on_message(): - return response + user_ws_streams = {stream_key: {} for stream_key in Constants.WS_SUB["USER_ORDERS_TRADES"]} + async with timeout(30): + await ws.subscribe(user_ws_streams) + async for response in ws.on_message(): + if ws.is_subscribed: + return True + return False def test_rest_auth(self): result = self.ev_loop.run_until_complete(self.rest_auth()) @@ -44,12 +50,5 @@ def test_rest_auth(self): assert "currency" in result[0].keys() def test_ws_auth(self): - try: - response = self.ev_loop.run_until_complete(self.ws_auth()) - no_errors = True - except Exception: - no_errors = False - assert no_errors is True - if 'result' not in response: - print(f"Unexpected response for API call: {response}") - assert response['result'] is True + subscribed = self.ev_loop.run_until_complete(self.ws_auth()) + assert subscribed is True diff --git a/test/connector/exchange/coinzoom/test_coinzoom_exchange.py b/test/connector/exchange/coinzoom/test_coinzoom_exchange.py index 6a968ed97a..7d158c981a 100644 --- a/test/connector/exchange/coinzoom/test_coinzoom_exchange.py +++ b/test/connector/exchange/coinzoom/test_coinzoom_exchange.py @@ -39,6 +39,7 @@ API_KEY = conf.coinzoom_api_key API_SECRET = conf.coinzoom_secret_key +API_USERNAME = conf.coinzoom_username class CoinzoomExchangeUnitTest(unittest.TestCase): @@ -68,6 +69,7 @@ def setUpClass(cls): cls.connector: CoinzoomExchange = CoinzoomExchange( coinzoom_api_key=API_KEY, coinzoom_secret_key=API_SECRET, + coinzoom_username=API_USERNAME, trading_pairs=[cls.trading_pair], trading_required=True ) @@ -351,7 +353,7 @@ def test_orders_saving_and_restoration(self): self.connector.remove_listener(event_tag, self.event_logger) # Clear the event loop self.event_logger.clear() - new_connector = CoinzoomExchange(API_KEY, API_SECRET, [self.trading_pair], True) + new_connector = CoinzoomExchange(API_KEY, API_SECRET, API_USERNAME, [self.trading_pair], True) for event_tag in self.events: new_connector.add_listener(event_tag, self.event_logger) recorder.stop() diff --git a/test/connector/exchange/coinzoom/test_coinzoom_user_stream_tracker.py b/test/connector/exchange/coinzoom/test_coinzoom_user_stream_tracker.py index 8a6337eaa5..f9f85f335d 100644 --- a/test/connector/exchange/coinzoom/test_coinzoom_user_stream_tracker.py +++ b/test/connector/exchange/coinzoom/test_coinzoom_user_stream_tracker.py @@ -20,13 +20,14 @@ class CoinzoomUserStreamTrackerUnitTest(unittest.TestCase): api_key = conf.coinzoom_api_key api_secret = conf.coinzoom_secret_key + api_username = conf.coinzoom_username @classmethod def setUpClass(cls): cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() cls.trading_pairs = ["BTC-USD"] cls.user_stream_tracker: CoinzoomUserStreamTracker = CoinzoomUserStreamTracker( - coinzoom_auth=CoinzoomAuth(cls.api_key, cls.api_secret), + coinzoom_auth=CoinzoomAuth(cls.api_key, cls.api_secret, cls.api_username), trading_pairs=cls.trading_pairs) cls.user_stream_tracker_task: asyncio.Task = safe_ensure_future(cls.user_stream_tracker.start()) From 24495552c5466fb6c0885c5bb32193ffbcf87062 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 24 Mar 2021 03:04:32 +0000 Subject: [PATCH 04/23] CoinZoom: Start work on Order Book CoinZoom: Start work on Order Book --- .../coinzoom_active_order_tracker.pxd | 2 + .../coinzoom_active_order_tracker.pyx | 96 ++++++++------ .../coinzoom_api_order_book_data_source.py | 74 +++++------ .../coinzoom_api_user_stream_data_source.py | 10 +- .../exchange/coinzoom/coinzoom_constants.py | 25 ++-- .../exchange/coinzoom/coinzoom_exchange.py | 123 ++++++++++-------- .../coinzoom/coinzoom_in_flight_order.py | 8 +- .../exchange/coinzoom/coinzoom_order_book.py | 24 ++-- .../coinzoom/coinzoom_order_book_message.py | 31 ++--- .../exchange/coinzoom/coinzoom_utils.py | 11 +- .../exchange/coinzoom/coinzoom_websocket.py | 21 ++- .../coinzoom/test_coinzoom_exchange.py | 4 +- .../test_coinzoom_order_book_tracker.py | 18 +-- 13 files changed, 237 insertions(+), 210 deletions(-) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd index 7990aaf2aa..752d47418a 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd @@ -4,6 +4,8 @@ cimport numpy as np cdef class CoinzoomActiveOrderTracker: cdef dict _active_bids cdef dict _active_asks + cdef dict _active_asks_ids + cdef dict _active_bids_ids cdef tuple c_convert_diff_message_to_np_arrays(self, object message) cdef tuple c_convert_snapshot_message_to_np_arrays(self, object message) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx index 8e4bb48d4a..418551652f 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx @@ -18,6 +18,8 @@ cdef class CoinzoomActiveOrderTracker: super().__init__() self._active_asks = active_asks or {} self._active_bids = active_bids or {} + self._active_asks_ids = {} + self._active_bids_ids = {} @classmethod def logger(cls) -> HummingbotLogger: @@ -44,7 +46,17 @@ cdef class CoinzoomActiveOrderTracker: def get_rates_and_quantities(self, entry) -> tuple: # price, quantity - return float(entry["price"]), float(entry["size"]) + return float(entry[0]), float(entry[1]) + + def get_rates_and_amts_with_ids(self, entry, id_list) -> tuple: + if len(entry) > 1: + price = float(entry[1]) + amount = float(entry[2]) + id_list[str(entry[0])] = price + else: + price = id_list.get(str(entry[0])) + amount = 0.0 + return price, amount cdef tuple c_convert_diff_message_to_np_arrays(self, object message): cdef: @@ -60,37 +72,32 @@ cdef class CoinzoomActiveOrderTracker: double timestamp = message.timestamp double amount = 0 - if "bid" in content_keys: - bid_entries = content["bid"] - if "ask" in content_keys: - ask_entries = content["ask"] - - bids = s_empty_diff - asks = s_empty_diff - - if len(bid_entries) > 0: - bids = np.array( - [[timestamp, - price, - amount, - message.update_id] - for price, amount in [self.get_rates_and_quantities(entry) for entry in bid_entries]], - dtype="float64", - ndmin=2 - ) - - if len(ask_entries) > 0: - asks = np.array( - [[timestamp, - price, - amount, - message.update_id] - for price, amount in [self.get_rates_and_quantities(entry) for entry in ask_entries]], - dtype="float64", - ndmin=2 - ) - - return bids, asks + if "b" in content_keys: + bid_entries = content["b"] + if "s" in content_keys: + ask_entries = content["s"] + + nps = { + 'bids': s_empty_diff, + 'asks': s_empty_diff, + } + + for entries, diff_key, id_list in [ + (content["b"], 'bids', self._active_bids_ids), + (content["s"], 'asks', self._active_asks_ids) + ]: + if len(entries) > 0: + nps[diff_key] = np.array( + [[timestamp, + price, + amount, + message.update_id] + for price, amount in [self.get_rates_and_amts_with_ids(entry, id_list) for entry in entries] if price is not None], + dtype="float64", + ndmin=2 + ) + + return nps['bids'], nps['asks'] cdef tuple c_convert_snapshot_message_to_np_arrays(self, object message): cdef: @@ -104,11 +111,21 @@ cdef class CoinzoomActiveOrderTracker: self._active_asks.clear() timestamp = message.timestamp content = message.content - - for snapshot_orders, active_orders in [(content["bid"], self._active_bids), (content["ask"], self._active_asks)]: - for entry in snapshot_orders: - price, amount = self.get_rates_and_quantities(entry) - active_orders[price] = amount + content_keys = list(content.keys()) + + if "bids" in content_keys: + for snapshot_orders, active_orders in [(content["bids"], self._active_bids), (content["asks"], self._active_asks)]: + for entry in snapshot_orders: + price, amount = self.get_rates_and_quantities(entry) + active_orders[price] = amount + else: + for snapshot_orders, active_orders, active_order_ids in [ + (content["b"], self._active_bids, self._active_bids_ids), + (content["s"], self._active_asks, self._active_asks_ids) + ]: + for entry in snapshot_orders: + price, amount = self.get_rates_and_amts_with_ids(entry, active_order_ids) + active_orders[price] = amount # Return the sorted snapshot tables. cdef: @@ -132,15 +149,16 @@ cdef class CoinzoomActiveOrderTracker: return bids, asks + # Is this method actually used? cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message): cdef: - double trade_type_value = 1.0 if message.content["side"] == "buy" else 2.0 + double trade_type_value = 1.0 if message.content[4] == "BUY" else 2.0 timestamp = message.timestamp content = message.content return np.array( - [timestamp, trade_type_value, float(content["price"]), float(content["quantity"])], + [timestamp, trade_type_value, float(content[1]), float(content[2])], dtype="float64" ) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_api_order_book_data_source.py b/hummingbot/connector/exchange/coinzoom/coinzoom_api_order_book_data_source.py index 56542705ac..49032d6f7f 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_api_order_book_data_source.py @@ -14,7 +14,6 @@ from .coinzoom_order_book import CoinzoomOrderBook from .coinzoom_websocket import CoinzoomWebsocket from .coinzoom_utils import ( - str_date_to_ts, convert_to_exchange_trading_pair, convert_from_exchange_trading_pair, api_call_with_retries, @@ -39,23 +38,18 @@ def __init__(self, trading_pairs: List[str] = None): @classmethod async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, Decimal]: results = {} - if len(trading_pairs) > 1: - tickers: List[Dict[Any]] = await api_call_with_retries("GET", Constants.ENDPOINT["TICKER"]) + tickers: List[Dict[Any]] = await api_call_with_retries("GET", Constants.ENDPOINT["TICKER"]) for trading_pair in trading_pairs: - ex_pair: str = convert_to_exchange_trading_pair(trading_pair) - if len(trading_pairs) > 1: - ticker: Dict[Any] = list([tic for tic in tickers if tic['symbol'] == ex_pair])[0] - else: - url_endpoint = Constants.ENDPOINT["TICKER_SINGLE"].format(trading_pair=ex_pair) - ticker: Dict[Any] = await api_call_with_retries("GET", url_endpoint) - results[trading_pair]: Decimal = Decimal(str(ticker["last"])) + ex_pair: str = convert_to_exchange_trading_pair(trading_pair, True) + ticker: Dict[Any] = list([tic for symbol, tic in tickers.items() if symbol == ex_pair])[0] + results[trading_pair]: Decimal = Decimal(str(ticker["last_price"])) return results @staticmethod async def fetch_trading_pairs() -> List[str]: try: symbols: List[Dict[str, Any]] = await api_call_with_retries("GET", Constants.ENDPOINT["SYMBOL"]) - trading_pairs: List[str] = list([convert_from_exchange_trading_pair(sym["id"]) for sym in symbols]) + trading_pairs: List[str] = list([convert_from_exchange_trading_pair(sym["symbol"]) for sym in symbols]) # Filter out unmatched pairs so nothing breaks return [sym for sym in trading_pairs if sym is not None] except Exception: @@ -69,10 +63,10 @@ async def get_order_book_data(trading_pair: str) -> Dict[str, any]: Get whole orderbook """ try: - ex_pair = convert_to_exchange_trading_pair(trading_pair) - orderbook_response: Dict[Any] = await api_call_with_retries("GET", Constants.ENDPOINT["ORDER_BOOK"], - params={"limit": 150, "symbols": ex_pair}) - return orderbook_response[ex_pair] + ex_pair = convert_to_exchange_trading_pair(trading_pair, True) + ob_endpoint = Constants.ENDPOINT["ORDER_BOOK"].format(trading_pair=ex_pair) + orderbook_response: Dict[Any] = await api_call_with_retries("GET", ob_endpoint) + return orderbook_response except CoinzoomAPIError as e: err = e.error_payload.get('error', e.error_payload) raise IOError( @@ -81,7 +75,7 @@ async def get_order_book_data(trading_pair: str) -> Dict[str, any]: async def get_new_order_book(self, trading_pair: str) -> OrderBook: snapshot: Dict[str, Any] = await self.get_order_book_data(trading_pair) - snapshot_timestamp: float = time.time() + snapshot_timestamp: float = float(snapshot['timestamp']) snapshot_msg: OrderBookMessage = CoinzoomOrderBook.snapshot_message_from_exchange( snapshot, snapshot_timestamp, @@ -102,30 +96,23 @@ async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asynci await ws.connect() for pair in self._trading_pairs: - await ws.subscribe(Constants.WS_SUB["TRADES"], convert_to_exchange_trading_pair(pair)) + await ws.subscribe({Constants.WS_SUB["TRADES"]: {'symbol': convert_to_exchange_trading_pair(pair)}}) async for response in ws.on_message(): - method: str = response.get("method", None) - trades_data: str = response.get("params", None) + msg_keys = list(response.keys()) if response is not None else [] - if trades_data is None or method != Constants.WS_METHODS['TRADES_UPDATE']: + if not Constants.WS_METHODS["TRADES_UPDATE"] in msg_keys: continue - pair: str = convert_from_exchange_trading_pair(response["params"]["symbol"]) - - for trade in trades_data["data"]: - trade: Dict[Any] = trade - trade_timestamp: int = str_date_to_ts(trade["timestamp"]) - trade_msg: OrderBookMessage = CoinzoomOrderBook.trade_message_from_exchange( - trade, - trade_timestamp, - metadata={"trading_pair": pair}) - output.put_nowait(trade_msg) + trade: List[Any] = response[Constants.WS_METHODS["TRADES_UPDATE"]] + trade_msg: OrderBookMessage = CoinzoomOrderBook.trade_message_from_exchange(trade) + output.put_nowait(trade_msg) except asyncio.CancelledError: raise except Exception: self.logger().error("Unexpected error.", exc_info=True) + raise await asyncio.sleep(5.0) finally: await ws.disconnect() @@ -145,17 +132,29 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp ] for pair in self._trading_pairs: - await ws.subscribe(Constants.WS_SUB["ORDERS"], convert_to_exchange_trading_pair(pair)) + ex_pair = convert_to_exchange_trading_pair(pair) + ws_stream = { + Constants.WS_SUB["ORDERS"]: { + 'requestId': ex_pair, + 'symbol': ex_pair, + 'aggregate': False, + 'depth': 0, + } + } + await ws.subscribe(ws_stream) async for response in ws.on_message(): - method: str = response.get("method", None) - order_book_data: str = response.get("params", None) + msg_keys = list(response.keys()) if response is not None else [] + + method_key = [key for key in msg_keys if key in order_book_methods] - if order_book_data is None or method not in order_book_methods: + if len(method_key) != 1: continue - timestamp: int = str_date_to_ts(order_book_data["timestamp"]) - pair: str = convert_from_exchange_trading_pair(order_book_data["symbol"]) + method: str = method_key[0] + order_book_data: dict = response + timestamp: int = int(time.time() * 1e3) + pair: str = convert_from_exchange_trading_pair(response[method]) order_book_msg_cls = (CoinzoomOrderBook.diff_message_from_exchange if method == Constants.WS_METHODS['ORDERS_UPDATE'] else @@ -187,10 +186,9 @@ async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, for trading_pair in self._trading_pairs: try: snapshot: Dict[str, any] = await self.get_order_book_data(trading_pair) - snapshot_timestamp: int = str_date_to_ts(snapshot["timestamp"]) snapshot_msg: OrderBookMessage = CoinzoomOrderBook.snapshot_message_from_exchange( snapshot, - snapshot_timestamp, + snapshot['timestamp'], metadata={"trading_pair": trading_pair} ) output.put_nowait(snapshot_msg) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py b/hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py index 38d7b704ff..41fdce9325 100755 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py @@ -52,20 +52,20 @@ async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: await self._ws.connect() - await self._ws.subscribe(Constants.WS_SUB["USER_ORDERS_TRADES"], None, {}) + await self._ws.subscribe({stream_key: {} for stream_key in Constants.WS_SUB["USER_ORDERS_TRADES"]}) event_methods = [ Constants.WS_METHODS["USER_ORDERS"], - Constants.WS_METHODS["USER_TRADES"], + Constants.WS_METHODS["USER_ORDERS_CANCEL"], ] async for msg in self._ws.on_message(): self._last_recv_time = time.time() - if msg.get("params", msg.get("result", None)) is None: + msg_keys = list(msg.keys()) if msg is not None else [] + + if not any(ws_method in msg_keys for ws_method in event_methods): continue - elif msg.get("method", None) in event_methods: - await self._ws_request_balances() yield msg except Exception as e: raise e diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py index f8cfcb2f31..db25704332 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py @@ -1,5 +1,10 @@ # A single source of truth for constant variables related to the exchange class Constants: + """ + API Documentation Links: + https://api-docs.coinzoom.com/ + https://api-markets.coinzoom.com/ + """ EXCHANGE_NAME = "coinzoom" REST_URL = "https://api.stage.coinzoom.com/api/v1/public" WS_PRIVATE_URL = "wss://api.stage.coinzoom.com/api/v1/public/market/data/stream" @@ -9,10 +14,9 @@ class Constants: ENDPOINT = { # Public Endpoints - "TICKER": "public/ticker", - "TICKER_SINGLE": "public/ticker/{trading_pair}", - "SYMBOL": "public/symbol", - "ORDER_BOOK": "public/orderbook", + "TICKER": "marketwatch/ticker", + "SYMBOL": "instruments", + "ORDER_BOOK": "marketwatch/orderbook/{trading_pair}/150/2", "ORDER_CREATE": "order", "ORDER_DELETE": "order/{id}", "ORDER_STATUS": "order/{id}", @@ -22,19 +26,18 @@ class Constants: WS_SUB = { "TRADES": "TradeSummaryRequest", - "ORDERS": "Orderbook", + "ORDERS": "OrderBookRequest", "USER_ORDERS_TRADES": ["OrderUpdateRequest"], } WS_METHODS = { - "ORDERS_SNAPSHOT": "snapshotOrderbook", - "ORDERS_UPDATE": "updateOrderbook", - "TRADES_SNAPSHOT": "snapshotTrades", - "TRADES_UPDATE": "updateTrades", + "ORDERS_SNAPSHOT": "ob", + "ORDERS_UPDATE": "oi", + "TRADES_UPDATE": "ts", "USER_BALANCE": "getTradingBalance", - "USER_ORDERS": "activeOrders", - "USER_TRADES": "report", + "USER_ORDERS": "OrderResponse", + "USER_ORDERS_CANCEL": "OrderCancelResponse", } # Timeouts diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py index 552dafe20f..206a613a11 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py @@ -227,9 +227,7 @@ async def check_network(self) -> NetworkStatus: """ try: # since there is no ping endpoint, the lowest rate call is to get BTC-USD symbol - await self._api_request("GET", - Constants.ENDPOINT['SYMBOL'], - params={'symbols': 'BTCUSD'}) + await self._api_request("GET", Constants.ENDPOINT['SYMBOL']) except asyncio.CancelledError: raise except Exception: @@ -274,29 +272,31 @@ def _format_trading_rules(self, symbols_info: Dict[str, Any]) -> Dict[str, Tradi Response Example: [ { - id: "BTCUSD", - baseCurrency: "BTC", - quoteCurrency: "USD", - quantityIncrement: "0.00001", - tickSize: "0.01", - takeLiquidityRate: "0.0025", - provideLiquidityRate: "0.001", - feeCurrency: "USD", - marginTrading: true, - maxInitialLeverage: "12.00" + "symbol" : "BTC/USD", + "baseCurrencyCode" : "BTC", + "termCurrencyCode" : "USD", + "minTradeAmt" : 0.0001, + "maxTradeAmt" : 10, + "maxPricePrecision" : 2, + "maxQuantityPrecision" : 6, + "issueOnly" : false } ] """ result = {} for rule in symbols_info: try: - trading_pair = convert_from_exchange_trading_pair(rule["id"]) - price_step = Decimal(str(rule["tickSize"])) - size_step = Decimal(str(rule["quantityIncrement"])) + trading_pair = convert_from_exchange_trading_pair(rule["symbol"]) + min_amount = Decimal(str(rule["minTradeAmt"])) + min_price = Decimal(f"1e-{rule['maxPricePrecision']}") result[trading_pair] = TradingRule(trading_pair, - min_order_size=size_step, - min_base_amount_increment=size_step, - min_price_increment=price_step) + min_order_size=min_amount, + max_order_size=Decimal(str(rule["maxTradeAmt"])), + min_price_increment=min_price, + min_base_amount_increment=min_amount, + min_notional_size=min(min_price * min_amount, Decimal("0.00000001")), + max_price_significant_digits=Decimal(str(rule["maxPricePrecision"])), + ) except Exception: self.logger().error(f"Error parsing the trading pair rule {rule}. Skipping.", exc_info=True) return result @@ -322,10 +322,9 @@ async def _api_request(self, qs_params: dict = params if method.upper() == "GET" else None req_form = aiohttp.FormData(params) if method.upper() == "POST" and params is not None else None # Generate auth headers if needed. - headers: dict = {"Content-Type": "application/x-www-form-urlencoded"} + headers: dict = {"Content-Type": "application/json"} if is_auth_required: - headers: dict = self._coinzoom_auth.get_headers(method, f"{Constants.REST_URL_AUTH}/{endpoint}", - params) + headers: dict = self._coinzoom_auth.get_headers() # Build request coro response_coro = shared_client.request(method=method.upper(), url=url, headers=headers, params=qs_params, data=req_form, @@ -623,23 +622,43 @@ def _process_order_message(self, order_msg: Dict[str, Any]): """ Updates in-flight order and triggers cancellation or failure event if needed. :param order_msg: The order response from either REST or web socket API (they are of the same format) - Example Order: - { - "id": "4345613661", - "clientOrderId": "57d5525562c945448e3cbd559bd068c3", - "symbol": "BCCBTC", - "side": "sell", - "status": "new", - "type": "limit", - "timeInForce": "GTC", - "quantity": "0.013", - "price": "0.100000", - "cumQuantity": "0.000", - "postOnly": false, - "createdAt": "2017-10-20T12:17:12.245Z", - "updatedAt": "2017-10-20T12:17:12.245Z", - "reportType": "status" - } + Example Orders: + Create: + { + 'orderId': '962a2a54-fbcf-4d89-8f37-a8854020a823', + 'symbol': 'BTC/USD', + 'orderType': 'LIMIT', + 'orderSide': 'BUY', + 'price': 5000, + 'quantity': 0.001, + 'executionType': 'NEW', + 'orderStatus': 'NEW', + 'lastQuantity': 0, + 'leavesQuantity': 0.001, + 'cumulativeQuantity': 0, + 'transactTime': '2021-03-23T19:06:41.621527Z' + } + Cancel Pending + { + 'orderId': '962a2a54-fbcf-4d89-8f37-a8854020a823', + 'response': 'Cancel Pending', + 'symbol': 'BTC/USD' + } + Cancelled + { + 'id': '4eb3f26c-91bd-4bd2-bacb-15b2f432c452', + 'orderId': '962a2a54-fbcf-4d89-8f37-a8854020a823', + 'symbol': 'BTC/USD', 'orderType': 'LIMIT', + 'orderSide': 'BUY', + 'price': 5000, + 'quantity': 0.001, + 'executionType': 'CANCEL', + 'orderStatus': 'CANCELLED', + 'lastQuantity': 0, + 'leavesQuantity': 0, + 'cumulativeQuantity': 0, + 'transactTime': '2021-03-23T19:06:51.155520Z' + } """ client_order_id = order_msg["clientOrderId"] if client_order_id not in self._in_flight_orders: @@ -831,22 +850,22 @@ async def _user_stream_event_listener(self): try: event_methods = [ Constants.WS_METHODS["USER_ORDERS"], - Constants.WS_METHODS["USER_TRADES"], + Constants.WS_METHODS["USER_ORDERS_CANCEL"], ] - method: str = event_message.get("method", None) - params: str = event_message.get("params", None) - account_balances: list = event_message.get("result", None) - if method not in event_methods and account_balances is None: - self.logger().error(f"Unexpected message in user stream: {event_message}.", exc_info=True) + msg_keys = list(event_message.keys()) if event_message is not None else [] + + method_key = [key for key in msg_keys if key in event_methods] + + if len(method_key) != 1: continue - if method == Constants.WS_METHODS["USER_TRADES"]: - await self._process_trade_message(params) - elif method == Constants.WS_METHODS["USER_ORDERS"]: - for order_msg in params: - self._process_order_message(order_msg) - elif isinstance(account_balances, list) and "currency" in account_balances[0]: - self._process_balance_message(account_balances) + + method: str = method_key[0] + + if method == Constants.WS_METHODS["USER_ORDERS"]: + self._process_order_message(event_message[method]) + elif method == Constants.WS_METHODS["USER_ORDERS_CANCEL"]: + self._process_order_message(event_message[method]) except asyncio.CancelledError: raise except Exception: diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py b/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py index d062d24923..83276c3f00 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py @@ -23,7 +23,7 @@ def __init__(self, trade_type: TradeType, price: Decimal, amount: Decimal, - initial_state: str = "new"): + initial_state: str = "NEW"): super().__init__( client_order_id, exchange_order_id, @@ -39,15 +39,15 @@ def __init__(self, @property def is_done(self) -> bool: - return self.last_state in {"filled", "canceled", "expired"} + return self.last_state in {"FILLED", "CANCELLED", "REJECTED"} @property def is_failure(self) -> bool: - return self.last_state in {"suspended"} + return self.last_state in {"REJECTED"} @property def is_cancelled(self) -> bool: - return self.last_state in {"canceled", "expired"} + return self.last_state in {"CANCELLED"} @classmethod def from_json(cls, data: Dict[str, Any]) -> InFlightOrderBase: diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book.py b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book.py index 96f00de856..e771d48cf3 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book.py @@ -14,6 +14,10 @@ OrderBookMessage, OrderBookMessageType ) from hummingbot.connector.exchange.coinzoom.coinzoom_order_book_message import CoinzoomOrderBookMessage +from .coinzoom_utils import ( + convert_from_exchange_trading_pair, + str_date_to_ts, +) _logger = None @@ -107,20 +111,18 @@ def trade_message_from_exchange(cls, :return: CoinzoomOrderBookMessage """ - if metadata: - msg.update(metadata) - - msg.update({ - "exchange_order_id": msg.get("id"), - "trade_type": msg.get("side"), - "price": msg.get("price"), - "amount": msg.get("quantity"), - }) + trade_msg = { + "trade_type": msg[4], + "price": msg[1], + "amount": msg[2], + "trading_pair": convert_from_exchange_trading_pair(msg[0]) + } + trade_timestamp = str_date_to_ts(msg[3]) return CoinzoomOrderBookMessage( message_type=OrderBookMessageType.TRADE, - content=msg, - timestamp=timestamp + content=trade_msg, + timestamp=trade_timestamp ) @classmethod diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py index 2eb6529a8c..4f91267868 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py @@ -2,18 +2,14 @@ from typing import ( Dict, - List, Optional, ) -from hummingbot.core.data_type.order_book_row import OrderBookRow from hummingbot.core.data_type.order_book_message import ( OrderBookMessage, OrderBookMessageType, ) -from .coinzoom_utils import ( - convert_from_exchange_trading_pair, -) +from hummingbot.connector.exchange.coinzoom.coinzoom_constants import Constants class CoinzoomOrderBookMessage(OrderBookMessage): @@ -37,38 +33,27 @@ def __new__( @property def update_id(self) -> int: if self.type in [OrderBookMessageType.DIFF, OrderBookMessageType.SNAPSHOT]: - return int(self.timestamp * 1e3) + return self.timestamp else: return -1 @property def trade_id(self) -> int: if self.type is OrderBookMessageType.TRADE: - return int(self.timestamp * 1e3) + return self.timestamp return -1 @property def trading_pair(self) -> str: - if "trading_pair" in self.content: - return self.content["trading_pair"] - elif "symbol" in self.content: - return convert_from_exchange_trading_pair(self.content["symbol"]) + return self.content["trading_pair"] @property - def asks(self) -> List[OrderBookRow]: - asks = map(self.content["ask"], lambda ask: {"price": ask["price"], "size": ask["size"]}) - - return [ - OrderBookRow(float(price), float(amount), self.update_id) for price, amount in asks - ] + def asks(self): + raise NotImplementedError(Constants.EXCHANGE_NAME + " order book uses active_order_tracker.") @property - def bids(self) -> List[OrderBookRow]: - bids = map(self.content["bid"], lambda bid: {"price": bid["price"], "size": bid["size"]}) - - return [ - OrderBookRow(float(price), float(amount), self.update_id) for price, amount in bids - ] + def bids(self): + raise NotImplementedError(Constants.EXCHANGE_NAME + " order book uses active_order_tracker.") def __eq__(self, other) -> bool: return self.type == other.type and self.timestamp == other.timestamp diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py index c9d285fca0..c3b4fd05de 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py @@ -18,7 +18,7 @@ EXAMPLE_PAIR = "BTC-USD" -DEFAULT_FEES = [0.1, 0.25] +DEFAULT_FEES = [0.2, 0.26] class CoinzoomAPIError(IOError): @@ -29,7 +29,7 @@ def __init__(self, error_payload: Dict[str, Any]): # convert date string to timestamp def str_date_to_ts(date: str) -> int: - return int(dateparse(date).timestamp()) + return int(dateparse(date).timestamp() * 1e3) # Request ID class @@ -49,9 +49,12 @@ def convert_from_exchange_trading_pair(ex_trading_pair: str) -> Optional[str]: return ex_trading_pair.replace("/", "-") -def convert_to_exchange_trading_pair(hb_trading_pair: str) -> str: +def convert_to_exchange_trading_pair(hb_trading_pair: str, alternative: bool = False) -> str: # CoinZoom uses uppercase (BTCUSDT) - return hb_trading_pair.replace("-", "/").upper() + if alternative: + return hb_trading_pair.replace("-", "_").upper() + else: + return hb_trading_pair.replace("-", "/").upper() def get_new_client_order_id(is_buy: bool, trading_pair: str) -> str: diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py b/hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py index 4764196258..1e233e2054 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_websocket.py @@ -66,12 +66,15 @@ async def _messages(self) -> AsyncIterable[Any]: raw_msg_str: str = await asyncio.wait_for(self._client.recv(), timeout=Constants.MESSAGE_TIMEOUT) try: msg = json.loads(raw_msg_str) - # DEBUG PRINTOUT - print(f"WS Msg: {msg}") - # DEBUG PRINTOUT + # CoinZoom doesn't support ping or heartbeat messages. # Can handle them here if that changes - use `safe_ensure_future`. - result: List[str] = list([d['result'] for k, d in msg.items() if d.get('result') is not None]) + + # Check response for a subscribed/unsubscribed message; + result: List[str] = list([d['result'] + for k, d in msg.items() + if (isinstance(d, dict) and d.get('result') is not None)]) + if len(result): if result[0] == 'subscribed': self._is_subscribed = True @@ -100,15 +103,7 @@ async def _emit(self, method: str, action: str, data: Optional[Dict[str, Any]] = **data } } - # payload = {**payload, **data} - - ws_data = json.dumps(payload) - # DEBUG PRINTOUT - print(f"WS Req: {ws_data}") - # DEBUG PRINTOUT - await self._client.send(json.dumps(payload)) - - return id + return await self._client.send(json.dumps(payload)) # request via websocket async def request(self, method: str, action: str, data: Optional[Dict[str, Any]] = {}) -> int: diff --git a/test/connector/exchange/coinzoom/test_coinzoom_exchange.py b/test/connector/exchange/coinzoom/test_coinzoom_exchange.py index 7d158c981a..51fcc36b0e 100644 --- a/test/connector/exchange/coinzoom/test_coinzoom_exchange.py +++ b/test/connector/exchange/coinzoom/test_coinzoom_exchange.py @@ -139,9 +139,9 @@ def _cancel_order(self, cl_order_id, connector=None): def test_estimate_fee(self): maker_fee = self.connector.estimate_fee_pct(True) - self.assertAlmostEqual(maker_fee, Decimal("0.001")) + self.assertAlmostEqual(maker_fee, Decimal("0.002")) taker_fee = self.connector.estimate_fee_pct(False) - self.assertAlmostEqual(taker_fee, Decimal("0.0025")) + self.assertAlmostEqual(taker_fee, Decimal("0.0026")) def test_buy_and_sell(self): price = self.connector.get_price(self.trading_pair, True) * Decimal("1.02") diff --git a/test/connector/exchange/coinzoom/test_coinzoom_order_book_tracker.py b/test/connector/exchange/coinzoom/test_coinzoom_order_book_tracker.py index 8aaed4c071..62a1a1b6d9 100755 --- a/test/connector/exchange/coinzoom/test_coinzoom_order_book_tracker.py +++ b/test/connector/exchange/coinzoom/test_coinzoom_order_book_tracker.py @@ -5,6 +5,7 @@ import asyncio import logging import unittest +from async_timeout import timeout from os.path import join, realpath from typing import Dict, Optional, List from hummingbot.core.event.event_logger import EventLogger @@ -38,11 +39,12 @@ def setUpClass(cls): @classmethod async def wait_til_tracker_ready(cls): - while True: - if len(cls.order_book_tracker.order_books) > 0: - print("Initialized real-time order books.") - return - await asyncio.sleep(1) + async with timeout(20): + while True: + if len(cls.order_book_tracker.order_books) > 0: + print("Initialized real-time order books.") + return + await asyncio.sleep(1) async def run_parallel_async(self, *tasks, timeout=None): future: asyncio.Future = asyncio.ensure_future(asyncio.gather(*tasks)) @@ -57,7 +59,7 @@ async def run_parallel_async(self, *tasks, timeout=None): return future.result() def run_parallel(self, *tasks): - return self.ev_loop.run_until_complete(self.run_parallel_async(*tasks)) + return self.ev_loop.run_until_complete(self.run_parallel_async(*tasks, timeout=60)) def setUp(self): self.event_logger = EventLogger() @@ -78,8 +80,8 @@ def test_order_book_trade_event_emission(self): self.assertTrue(type(ob_trade_event.amount) == float) self.assertTrue(type(ob_trade_event.price) == float) self.assertTrue(type(ob_trade_event.type) == TradeType) - # datetime is in seconds - self.assertTrue(math.ceil(math.log10(ob_trade_event.timestamp)) == 10) + # datetime is in milliseconds + self.assertTrue(math.ceil(math.log10(ob_trade_event.timestamp)) == 13) self.assertTrue(ob_trade_event.amount > 0) self.assertTrue(ob_trade_event.price > 0) From 563d745bc699836cd90044c984a50fbc3e7bbd3c Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 24 Mar 2021 03:04:27 +0000 Subject: [PATCH 05/23] CoinZoom: Work on order creation/status --- .../exchange/coinzoom/coinzoom_constants.py | 6 +- .../exchange/coinzoom/coinzoom_exchange.py | 188 +++++++++--------- .../coinzoom/coinzoom_in_flight_order.py | 111 +++++++---- 3 files changed, 172 insertions(+), 133 deletions(-) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py index db25704332..a56a09b0f4 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py @@ -17,9 +17,9 @@ class Constants: "TICKER": "marketwatch/ticker", "SYMBOL": "instruments", "ORDER_BOOK": "marketwatch/orderbook/{trading_pair}/150/2", - "ORDER_CREATE": "order", - "ORDER_DELETE": "order/{id}", - "ORDER_STATUS": "order/{id}", + "ORDER_CREATE": "orders/new", + "ORDER_DELETE": "orders/cancel", + "ORDER_STATUS": "orders/list", "USER_ORDERS": "order", "USER_BALANCES": "ledger/list", } diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py index 206a613a11..c0459fe493 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py @@ -11,6 +11,7 @@ import aiohttp import math import time +import ujson from async_timeout import timeout from hummingbot.core.network_iterator import NetworkStatus @@ -320,14 +321,14 @@ async def _api_request(self, shared_client = await self._http_client() # Turn `params` into either GET params or POST body data qs_params: dict = params if method.upper() == "GET" else None - req_form = aiohttp.FormData(params) if method.upper() == "POST" and params is not None else None + req_params = ujson.dumps(params) if method.upper() == "POST" and params is not None else None # Generate auth headers if needed. headers: dict = {"Content-Type": "application/json"} if is_auth_required: headers: dict = self._coinzoom_auth.get_headers() # Build request coro response_coro = shared_client.request(method=method.upper(), url=url, headers=headers, - params=qs_params, data=req_form, + params=qs_params, data=req_params, timeout=Constants.API_CALL_TIMEOUT) http_status, parsed_response, request_errors = await aiohttp_response_with_errors(response_coro) if request_errors or parsed_response is None: @@ -429,22 +430,19 @@ async def _create_order(self, if amount < trading_rule.min_order_size: raise ValueError(f"Buy order amount {amount} is lower than the minimum order size " f"{trading_rule.min_order_size}.") - order_type_str = order_type.name.lower().split("_")[0] + order_type_str = order_type.name.upper().split("_")[0] api_params = {"symbol": convert_to_exchange_trading_pair(trading_pair), - "side": trade_type.name.lower(), - "type": order_type_str, - "price": f"{price:f}", + "orderType": order_type_str, + "orderSide": trade_type.name.upper(), "quantity": f"{amount:f}", - "clientOrderId": order_id, - # Without strict validate, CoinZoom might adjust order prices/sizes. - "strictValidate": "true", + "price": f"{price:f}", + # "clientOrderId": order_id, + "payFeesWithZoomToken": "true", } - if order_type is OrderType.LIMIT_MAKER: - api_params["postOnly"] = "true" self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, order_type) try: order_result = await self._api_request("POST", Constants.ENDPOINT["ORDER_CREATE"], api_params, True) - exchange_order_id = str(order_result["id"]) + exchange_order_id = str(order_result) tracked_order = self._in_flight_orders.get(order_id) if tracked_order is not None: self.logger().info(f"Created {order_type.name} {trade_type.name} order {order_id} for " @@ -517,15 +515,21 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: raise ValueError(f"Failed to cancel order - {order_id}. Order not found.") if tracked_order.exchange_order_id is None: await tracked_order.get_exchange_order_id() - # ex_order_id = tracked_order.exchange_order_id - await self._api_request("DELETE", - Constants.ENDPOINT["ORDER_DELETE"].format(id=order_id), + ex_order_id = tracked_order.exchange_order_id + api_params = { + "orderId": ex_order_id, + "symbol": convert_to_exchange_trading_pair(trading_pair) + } + await self._api_request("POST", + Constants.ENDPOINT["ORDER_DELETE"], + api_params, is_auth_required=True) order_was_cancelled = True except asyncio.CancelledError: raise except CoinzoomAPIError as e: err = e.error_payload.get('error', e.error_payload) + print(f"order cancel error: {err}") self._order_not_found_records[order_id] = self._order_not_found_records.get(order_id, 0) + 1 if err.get('code') == 20002 and \ self._order_not_found_records[order_id] >= self.ORDER_NOT_EXIST_CANCEL_COUNT: @@ -586,19 +590,23 @@ async def _update_order_status(self): if current_tick > last_tick and len(self._in_flight_orders) > 0: tracked_orders = list(self._in_flight_orders.values()) - tasks = [] - for tracked_order in tracked_orders: - # exchange_order_id = await tracked_order.get_exchange_order_id() - order_id = tracked_order.client_order_id - tasks.append(self._api_request("GET", - Constants.ENDPOINT["ORDER_STATUS"].format(id=order_id), - is_auth_required=True)) - self.logger().debug(f"Polling for order status updates of {len(tasks)} orders.") - responses = await safe_gather(*tasks, return_exceptions=True) - for response, tracked_order in zip(responses, tracked_orders): + api_params = { + 'symbol': None, + 'orderSide': None, + 'orderStatuses': ["NEW", "PARTIALLY_FILLED"], + 'size': 500, + 'bookmarkOrderId': None + } + self.logger().debug(f"Polling for order status updates of {len(tracked_orders)} orders.") + open_orders = await self._api_request("POST", + Constants.ENDPOINT["ORDER_STATUS"], + api_params, + is_auth_required=True) + for response, tracked_order in zip(open_orders, tracked_orders): client_order_id = tracked_order.client_order_id if isinstance(response, CoinzoomAPIError): err = response.error_payload.get('error', response.error_payload) + print(f"order update err {err}") if err.get('code') == 20002: self._order_not_found_records[client_order_id] = \ self._order_not_found_records.get(client_order_id, 0) + 1 @@ -612,8 +620,8 @@ async def _update_order_status(self): self.stop_tracking_order(client_order_id) else: continue - elif "clientOrderId" not in response: - self.logger().info(f"_update_order_status clientOrderId not in resp: {response}") + elif "id" not in response: + self.logger().info(f"_update_order_status id not in resp: {response}") continue else: self._process_order_message(response) @@ -623,28 +631,35 @@ def _process_order_message(self, order_msg: Dict[str, Any]): Updates in-flight order and triggers cancellation or failure event if needed. :param order_msg: The order response from either REST or web socket API (they are of the same format) Example Orders: - Create: + REST request { - 'orderId': '962a2a54-fbcf-4d89-8f37-a8854020a823', - 'symbol': 'BTC/USD', - 'orderType': 'LIMIT', - 'orderSide': 'BUY', - 'price': 5000, - 'quantity': 0.001, - 'executionType': 'NEW', - 'orderStatus': 'NEW', - 'lastQuantity': 0, - 'leavesQuantity': 0.001, - 'cumulativeQuantity': 0, - 'transactTime': '2021-03-23T19:06:41.621527Z' - } - Cancel Pending - { - 'orderId': '962a2a54-fbcf-4d89-8f37-a8854020a823', - 'response': 'Cancel Pending', - 'symbol': 'BTC/USD' + "id" : "977f82aa-23dc-4c8b-982c-2ee7d2002882", + "clientOrderId" : null, + "symbol" : "BTC/USD", + "orderType" : "LIMIT", + "orderSide" : "BUY", + "quantity" : 0.1, + "price" : 54570, + "payFeesWithZoomToken" : false, + "orderStatus" : "PARTIALLY_FILLED", + "timestamp" : "2021-03-24T04:07:26.260253Z", + "executions" : + [ + { + "id" : "38761582-2b37-4e27-a561-434981d21a96", + "executionType" : "PARTIAL_FILL", + "orderStatus" : "PARTIALLY_FILLED", + "lastPrice" : 54570, + "averagePrice" : 54570, + "lastQuantity" : 0.01, + "leavesQuantity" : 0.09, + "cumulativeQuantity" : 0.01, + "rejectReason" : null, + "timestamp" : "2021-03-24T04:07:44.503222Z" + } + ] } - Cancelled + WS request { 'id': '4eb3f26c-91bd-4bd2-bacb-15b2f432c452', 'orderId': '962a2a54-fbcf-4d89-8f37-a8854020a823', @@ -660,17 +675,34 @@ def _process_order_message(self, order_msg: Dict[str, Any]): 'transactTime': '2021-03-23T19:06:51.155520Z' } """ - client_order_id = order_msg["clientOrderId"] - if client_order_id not in self._in_flight_orders: - return - tracked_order = self._in_flight_orders[client_order_id] + if order_msg.get('clientOrderId') is not None: + client_order_id = order_msg["clientOrderId"] + if client_order_id not in self._in_flight_orders: + return + tracked_order = self._in_flight_orders[client_order_id] + else: + if "orderId" not in order_msg: + exchange_order_id = str(order_msg["id"]) + else: + exchange_order_id = str(order_msg["orderId"]) + tracked_orders = list(self._in_flight_orders.values()) + track_order = [o for o in tracked_orders if exchange_order_id == o.exchange_order_id] + if not track_order: + return + tracked_order = track_order[0] # Update order execution status - tracked_order.last_state = order_msg["status"] + tracked_order.last_state = order_msg["orderStatus"] # update order - tracked_order.executed_amount_base = Decimal(order_msg["cumQuantity"]) - tracked_order.executed_amount_quote = Decimal(order_msg["price"]) * Decimal(order_msg["cumQuantity"]) + tracked_order.executed_amount_base = Decimal(order_msg["cumulativeQuantity"]) + tracked_order.executed_amount_quote = Decimal(order_msg["price"]) * Decimal(order_msg["cumulativeQuantity"]) + + # Estimate fee + order_msg["trade_fee"] = self.estimate_fee_pct(tracked_order.order_type is OrderType.LIMIT_MAKER) + updated = tracked_order.update_with_order_update(order_msg) - if tracked_order.is_cancelled: + if updated: + safe_ensure_future(self._trigger_order_fill(tracked_order, order_msg)) + elif tracked_order.is_cancelled: self.logger().info(f"Successfully cancelled order {client_order_id}.") self.stop_tracking_order(client_order_id) self.trigger_event(MarketEvent.OrderCancelled, @@ -683,42 +715,9 @@ def _process_order_message(self, order_msg: Dict[str, Any]): self.current_timestamp, client_order_id, tracked_order.order_type)) self.stop_tracking_order(client_order_id) - async def _process_trade_message(self, trade_msg: Dict[str, Any]): - """ - Updates in-flight order and trigger order filled event for trade message received. Triggers order completed - event if the total executed amount equals to the specified order amount. - Example Trade: - { - "id": "4345697765", - "clientOrderId": "53b7cf917963464a811a4af426102c19", - "symbol": "ETHBTC", - "side": "sell", - "status": "filled", - "type": "limit", - "timeInForce": "GTC", - "quantity": "0.001", - "price": "0.053868", - "cumQuantity": "0.001", - "postOnly": false, - "createdAt": "2017-10-20T12:20:05.952Z", - "updatedAt": "2017-10-20T12:20:38.708Z", - "reportType": "trade", - "tradeQuantity": "0.001", - "tradePrice": "0.053868", - "tradeId": 55051694, - "tradeFee": "-0.000000005" - } - """ - tracked_orders = list(self._in_flight_orders.values()) - for order in tracked_orders: - await order.get_exchange_order_id() - track_order = [o for o in tracked_orders if trade_msg["id"] == o.exchange_order_id] - if not track_order: - return - tracked_order = track_order[0] - updated = tracked_order.update_with_trade_update(trade_msg) - if not updated: - return + async def _trigger_order_fill(self, + tracked_order: CoinzoomInFlightOrder, + update_msg: Dict[str, Any]): self.trigger_event( MarketEvent.OrderFilled, OrderFilledEvent( @@ -727,10 +726,9 @@ async def _process_trade_message(self, trade_msg: Dict[str, Any]): tracked_order.trading_pair, tracked_order.trade_type, tracked_order.order_type, - Decimal(str(trade_msg.get("tradePrice", "0"))), - Decimal(str(trade_msg.get("tradeQuantity", "0"))), - TradeFee(0.0, [(tracked_order.quote_asset, Decimal(str(trade_msg.get("tradeFee", "0"))))]), - exchange_trade_id=trade_msg["id"] + Decimal(str(update_msg.get("price", "0"))), + tracked_order.executed_amount_base, + TradeFee(percent=update_msg["trade_fee"]), ) ) if math.isclose(tracked_order.executed_amount_base, tracked_order.amount) or \ diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py b/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py index 83276c3f00..cf756769c0 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py @@ -72,47 +72,88 @@ def from_json(cls, data: Dict[str, Any]) -> InFlightOrderBase: retval.last_state = data["last_state"] return retval - def update_with_trade_update(self, trade_update: Dict[str, Any]) -> bool: + def update_with_order_update(self, order_update: Dict[str, Any]) -> bool: """ - Updates the in flight order with trade update (from private/get-order-detail end point) + Updates the in flight order with order update (from private/get-order-detail end point) return: True if the order gets updated otherwise False - Example Trade: - { - "id": "4345697765", - "clientOrderId": "53b7cf917963464a811a4af426102c19", - "symbol": "ETHBTC", - "side": "sell", - "status": "filled", - "type": "limit", - "timeInForce": "GTC", - "quantity": "0.001", - "price": "0.053868", - "cumQuantity": "0.001", - "postOnly": false, - "createdAt": "2017-10-20T12:20:05.952Z", - "updatedAt": "2017-10-20T12:20:38.708Z", - "reportType": "trade", - } - ... Trade variables are only included after fills. - { - "tradeQuantity": "0.001", - "tradePrice": "0.053868", - "tradeId": 55051694, - "tradeFee": "-0.000000005" - } + Example Orders: + REST request + { + "id" : "977f82aa-23dc-4c8b-982c-2ee7d2002882", + "clientOrderId" : null, + "symbol" : "BTC/USD", + "orderType" : "LIMIT", + "orderSide" : "BUY", + "quantity" : 0.1, + "price" : 54570, + "payFeesWithZoomToken" : false, + "orderStatus" : "PARTIALLY_FILLED", + "timestamp" : "2021-03-24T04:07:26.260253Z", + "executions" : + [ + { + "id" : "38761582-2b37-4e27-a561-434981d21a96", + "executionType" : "PARTIAL_FILL", + "orderStatus" : "PARTIALLY_FILLED", + "lastPrice" : 54570, + "averagePrice" : 54570, + "lastQuantity" : 0.01, + "leavesQuantity" : 0.09, + "cumulativeQuantity" : 0.01, + "rejectReason" : null, + "timestamp" : "2021-03-24T04:07:44.503222Z" + } + ] + } + WS request + { + 'id': '4eb3f26c-91bd-4bd2-bacb-15b2f432c452', + 'orderId': '962a2a54-fbcf-4d89-8f37-a8854020a823', + 'symbol': 'BTC/USD', 'orderType': 'LIMIT', + 'orderSide': 'BUY', + 'price': 5000, + 'quantity': 0.001, + 'executionType': 'CANCEL', + 'orderStatus': 'CANCELLED', + 'lastQuantity': 0, + 'leavesQuantity': 0, + 'cumulativeQuantity': 0, + 'transactTime': '2021-03-23T19:06:51.155520Z' + } """ - self.executed_amount_base = Decimal(str(trade_update["cumQuantity"])) + if 'cumulativeQuantity' not in order_update and 'executions' not in order_update: + return False + + trades = order_update.get('executions') + if trades is not None: + new_trades = False + for trade in trades: + trade_id = str(trade["timestamp"]) + if trade_id not in self.trade_id_set: + self.trade_id_set.add(trade_id) + # Add executed amounts + executed_price = Decimal(str(trade.get("lastPrice", "0"))) + self.executed_amount_base += Decimal(str(trade["lastQuantity"])) + self.executed_amount_quote += executed_price * self.executed_amount_base + # Set new trades flag + new_trades = True + if not new_trades: + # trades already recorded + return False + else: + trade_id = str(order_update["transactTime"]) + if trade_id in self.trade_id_set: + # trade already recorded + return False + self.trade_id_set.add(trade_id) + # Set executed amounts + executed_price = Decimal(str(order_update.get("price", "0"))) + self.executed_amount_base = Decimal(str(order_update["cumulativeQuantity"])) + self.executed_amount_quote = executed_price * self.executed_amount_base if self.executed_amount_base <= s_decimal_0: # No trades executed yet. return False - trade_id = trade_update["updatedAt"] - if trade_id in self.trade_id_set: - # trade already recorded - return False - self.trade_id_set.add(trade_id) - self.fee_paid += Decimal(str(trade_update.get("tradeFee", "0"))) - self.executed_amount_quote += (Decimal(str(trade_update.get("tradePrice", "0"))) * - Decimal(str(trade_update.get("tradeQuantity", "0")))) + self.fee_paid += order_update.get("trade_fee") * self.executed_amount_base if not self.fee_asset: self.fee_asset = self.quote_asset return True From 6842f87c8829359eb65033d5e70b57a88b71f63a Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 24 Mar 2021 15:59:09 +0000 Subject: [PATCH 06/23] CoinZoom: More work on order creation and status --- .../coinzoom_active_order_tracker.pyx | 42 ++----- .../coinzoom_api_user_stream_data_source.py | 5 +- .../exchange/coinzoom/coinzoom_constants.py | 6 +- .../exchange/coinzoom/coinzoom_exchange.py | 109 ++++++++++-------- .../coinzoom/coinzoom_in_flight_order.py | 5 +- .../exchange/coinzoom/coinzoom_utils.py | 2 +- .../coinzoom/test_coinzoom_exchange.py | 24 ++-- 7 files changed, 99 insertions(+), 94 deletions(-) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx index 418551652f..001f1d4c1c 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx @@ -71,32 +71,24 @@ cdef class CoinzoomActiveOrderTracker: dict order_dict double timestamp = message.timestamp double amount = 0 + dict nps = {'bids': s_empty_diff, 'asks': s_empty_diff} if "b" in content_keys: bid_entries = content["b"] if "s" in content_keys: ask_entries = content["s"] - nps = { - 'bids': s_empty_diff, - 'asks': s_empty_diff, - } - for entries, diff_key, id_list in [ - (content["b"], 'bids', self._active_bids_ids), - (content["s"], 'asks', self._active_asks_ids) + (bid_entries, 'bids', self._active_bids_ids), + (ask_entries, 'asks', self._active_asks_ids) ]: if len(entries) > 0: nps[diff_key] = np.array( - [[timestamp, - price, - amount, - message.update_id] - for price, amount in [self.get_rates_and_amts_with_ids(entry, id_list) for entry in entries] if price is not None], - dtype="float64", - ndmin=2 + [[timestamp, price, amount, message.update_id] + for price, amount in [self.get_rates_and_amts_with_ids(entry, id_list) for entry in entries] + if price is not None], + dtype="float64", ndmin=2 ) - return nps['bids'], nps['asks'] cdef tuple c_convert_snapshot_message_to_np_arrays(self, object message): @@ -130,16 +122,10 @@ cdef class CoinzoomActiveOrderTracker: # Return the sorted snapshot tables. cdef: np.ndarray[np.float64_t, ndim=2] bids = np.array( - [[message.timestamp, - float(price), - float(self._active_bids[price]), - message.update_id] + [[message.timestamp, float(price), float(self._active_bids[price]), message.update_id] for price in sorted(self._active_bids.keys())], dtype='float64', ndmin=2) np.ndarray[np.float64_t, ndim=2] asks = np.array( - [[message.timestamp, - float(price), - float(self._active_asks[price]), - message.update_id] + [[message.timestamp, float(price), float(self._active_asks[price]), message.update_id] for price in sorted(self._active_asks.keys(), reverse=True)], dtype='float64', ndmin=2) if bids.shape[1] != 4: @@ -153,14 +139,10 @@ cdef class CoinzoomActiveOrderTracker: cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message): cdef: double trade_type_value = 1.0 if message.content[4] == "BUY" else 2.0 + list content = message.content - timestamp = message.timestamp - content = message.content - - return np.array( - [timestamp, trade_type_value, float(content[1]), float(content[2])], - dtype="float64" - ) + return np.array([message.timestamp, trade_type_value, float(content[1]), float(content[2])], + dtype="float64") def convert_diff_message_to_order_book_row(self, message): np_bids, np_asks = self.c_convert_diff_message_to_np_arrays(message) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py b/hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py index 41fdce9325..7aede77e89 100755 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_api_user_stream_data_source.py @@ -52,11 +52,12 @@ async def _listen_to_orders_trades_balances(self) -> AsyncIterable[Any]: await self._ws.connect() - await self._ws.subscribe({stream_key: {} for stream_key in Constants.WS_SUB["USER_ORDERS_TRADES"]}) + await self._ws.subscribe({Constants.WS_SUB["USER_ORDERS_TRADES"]: {}}) event_methods = [ Constants.WS_METHODS["USER_ORDERS"], - Constants.WS_METHODS["USER_ORDERS_CANCEL"], + # We don't need to know about pending cancels + # Constants.WS_METHODS["USER_ORDERS_CANCEL"], ] async for msg in self._ws.on_message(): diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py index a56a09b0f4..d527b4e986 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py @@ -10,7 +10,7 @@ class Constants: WS_PRIVATE_URL = "wss://api.stage.coinzoom.com/api/v1/public/market/data/stream" WS_PUBLIC_URL = "wss://api.stage.coinzoom.com/api/v1/public/market/data/stream" - HBOT_BROKER_ID = "refzzz48" + HBOT_BROKER_ID = "CZ_API_HBOT" ENDPOINT = { # Public Endpoints @@ -20,14 +20,14 @@ class Constants: "ORDER_CREATE": "orders/new", "ORDER_DELETE": "orders/cancel", "ORDER_STATUS": "orders/list", - "USER_ORDERS": "order", + "USER_ORDERS": "orders/list", "USER_BALANCES": "ledger/list", } WS_SUB = { "TRADES": "TradeSummaryRequest", "ORDERS": "OrderBookRequest", - "USER_ORDERS_TRADES": ["OrderUpdateRequest"], + "USER_ORDERS_TRADES": "OrderUpdateRequest", } diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py index c0459fe493..ab7c69be4f 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py @@ -436,6 +436,9 @@ async def _create_order(self, "orderSide": trade_type.name.upper(), "quantity": f"{amount:f}", "price": f"{price:f}", + # Waiting for changes to CoinZoom API for this one. + # "originType": Constants.HBOT_BROKER_ID, + # CoinZoom doesn't support client order id yet # "clientOrderId": order_id, "payFeesWithZoomToken": "true", } @@ -530,10 +533,11 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: except CoinzoomAPIError as e: err = e.error_payload.get('error', e.error_payload) print(f"order cancel error: {err}") - self._order_not_found_records[order_id] = self._order_not_found_records.get(order_id, 0) + 1 - if err.get('code') == 20002 and \ - self._order_not_found_records[order_id] >= self.ORDER_NOT_EXIST_CANCEL_COUNT: - order_was_cancelled = True + # TODO: Still need to handle order cancel errors. + # self._order_not_found_records[order_id] = self._order_not_found_records.get(order_id, 0) + 1 + # if err.get('code') == 20002 and \ + # self._order_not_found_records[order_id] >= self.ORDER_NOT_EXIST_CANCEL_COUNT: + # order_was_cancelled = True if order_was_cancelled: self.logger().info(f"Successfully cancelled order {order_id} on {Constants.EXCHANGE_NAME}.") self.stop_tracking_order(order_id) @@ -602,29 +606,25 @@ async def _update_order_status(self): Constants.ENDPOINT["ORDER_STATUS"], api_params, is_auth_required=True) - for response, tracked_order in zip(open_orders, tracked_orders): + + open_orders_dict = {o['id']: o for o in open_orders} + found_ex_order_ids = list(open_orders_dict.keys()) + + for tracked_order in tracked_orders: client_order_id = tracked_order.client_order_id - if isinstance(response, CoinzoomAPIError): - err = response.error_payload.get('error', response.error_payload) - print(f"order update err {err}") - if err.get('code') == 20002: - self._order_not_found_records[client_order_id] = \ - self._order_not_found_records.get(client_order_id, 0) + 1 - if self._order_not_found_records[client_order_id] < self.ORDER_NOT_EXIST_CONFIRMATION_COUNT: - # Wait until the order not found error have repeated a few times before actually treating - # it as failed. See: https://github.com/CoinAlpha/hummingbot/issues/601 - continue - self.trigger_event(MarketEvent.OrderFailure, - MarketOrderFailureEvent( - self.current_timestamp, client_order_id, tracked_order.order_type)) - self.stop_tracking_order(client_order_id) - else: + ex_order_id = tracked_order.exchange_order_id + if ex_order_id not in found_ex_order_ids: + self._order_not_found_records[client_order_id] = \ + self._order_not_found_records.get(client_order_id, 0) + 1 + if self._order_not_found_records[client_order_id] < self.ORDER_NOT_EXIST_CONFIRMATION_COUNT: + # Wait until the order is not found a few times before actually treating it as failed. continue - elif "id" not in response: - self.logger().info(f"_update_order_status id not in resp: {response}") - continue + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent( + self.current_timestamp, client_order_id, tracked_order.order_type)) + self.stop_tracking_order(client_order_id) else: - self._process_order_message(response) + self._process_order_message(open_orders_dict[ex_order_id]) def _process_order_message(self, order_msg: Dict[str, Any]): """ @@ -661,7 +661,6 @@ def _process_order_message(self, order_msg: Dict[str, Any]): } WS request { - 'id': '4eb3f26c-91bd-4bd2-bacb-15b2f432c452', 'orderId': '962a2a54-fbcf-4d89-8f37-a8854020a823', 'symbol': 'BTC/USD', 'orderType': 'LIMIT', 'orderSide': 'BUY', @@ -673,6 +672,13 @@ def _process_order_message(self, order_msg: Dict[str, Any]): 'leavesQuantity': 0, 'cumulativeQuantity': 0, 'transactTime': '2021-03-23T19:06:51.155520Z' + + ... Optional fields + + 'id': '4eb3f26c-91bd-4bd2-bacb-15b2f432c452', + "orderType": "LIMIT", + "lastPrice": 56518.7, + "averagePrice": 56518.7, } """ if order_msg.get('clientOrderId') is not None: @@ -690,11 +696,6 @@ def _process_order_message(self, order_msg: Dict[str, Any]): if not track_order: return tracked_order = track_order[0] - # Update order execution status - tracked_order.last_state = order_msg["orderStatus"] - # update order - tracked_order.executed_amount_base = Decimal(order_msg["cumulativeQuantity"]) - tracked_order.executed_amount_quote = Decimal(order_msg["price"]) * Decimal(order_msg["cumulativeQuantity"]) # Estimate fee order_msg["trade_fee"] = self.estimate_fee_pct(tracked_order.order_type is OrderType.LIMIT_MAKER) @@ -703,17 +704,17 @@ def _process_order_message(self, order_msg: Dict[str, Any]): if updated: safe_ensure_future(self._trigger_order_fill(tracked_order, order_msg)) elif tracked_order.is_cancelled: - self.logger().info(f"Successfully cancelled order {client_order_id}.") - self.stop_tracking_order(client_order_id) + self.logger().info(f"Successfully cancelled order {tracked_order.client_order_id}.") + self.stop_tracking_order(tracked_order.client_order_id) self.trigger_event(MarketEvent.OrderCancelled, - OrderCancelledEvent(self.current_timestamp, client_order_id)) + OrderCancelledEvent(self.current_timestamp, tracked_order.client_order_id)) tracked_order.cancelled_event.set() elif tracked_order.is_failure: - self.logger().info(f"The market order {client_order_id} has failed according to order status API. ") + self.logger().info(f"The order {tracked_order.client_order_id} has failed according to order status API. ") self.trigger_event(MarketEvent.OrderFailure, MarketOrderFailureEvent( - self.current_timestamp, client_order_id, tracked_order.order_type)) - self.stop_tracking_order(client_order_id) + self.current_timestamp, tracked_order.client_order_id, tracked_order.order_type)) + self.stop_tracking_order(tracked_order.client_order_id) async def _trigger_order_fill(self, tracked_order: CoinzoomInFlightOrder, @@ -726,7 +727,7 @@ async def _trigger_order_fill(self, tracked_order.trading_pair, tracked_order.trade_type, tracked_order.order_type, - Decimal(str(update_msg.get("price", "0"))), + Decimal(str(update_msg.get("averagePrice", update_msg.get("price", "0")))), tracked_order.executed_amount_base, TradeFee(percent=update_msg["trade_fee"]), ) @@ -777,8 +778,8 @@ async def cancel_all(self, timeout_seconds: float) -> List[CancellationResult]: :param timeout_seconds: The timeout at which the operation will be canceled. :returns List of CancellationResult which indicates whether each order is successfully cancelled. """ - if self._trading_pairs is None: - raise Exception("cancel_all can only be used when trading_pairs are specified.") + # if self._trading_pairs is None: + # raise Exception("cancel_all can only be used when trading_pairs are specified.") open_orders = [o for o in self._in_flight_orders.values() if not o.is_done] if len(open_orders) == 0: return [] @@ -872,25 +873,41 @@ async def _user_stream_event_listener(self): # This is currently unused, but looks like a future addition. async def get_open_orders(self) -> List[OpenOrder]: - result = await self._api_request("GET", Constants.ENDPOINT["USER_ORDERS"], is_auth_required=True) + tracked_orders = list(self._in_flight_orders.values()) + api_params = { + 'symbol': None, + 'orderSide': None, + 'orderStatuses': ["NEW", "PARTIALLY_FILLED"], + 'size': 500, + 'bookmarkOrderId': None + } + result = await self._api_request("POST", Constants.ENDPOINT["USER_ORDERS"], api_params, is_auth_required=True) ret_val = [] for order in result: - if Constants.HBOT_BROKER_ID not in order["clientOrderId"]: + exchange_order_id = str(order["id"]) + # CoinZoom doesn't support client order ids yet so we must find it from the tracked orders. + track_order = [o for o in tracked_orders if exchange_order_id == o.exchange_order_id] + if not track_order or len(track_order) < 1: + # Skip untracked orders continue - if order["type"] != OrderType.LIMIT.name.lower(): + client_order_id = track_order[0].client_order_id + # if Constants.HBOT_BROKER_ID not in order["clientOrderId"]: + # continue + if order["orderType"] != OrderType.LIMIT.name.upper(): self.logger().info(f"Unsupported order type found: {order['type']}") + # Skip and report non-limit orders continue ret_val.append( OpenOrder( - client_order_id=order["clientOrderId"], + client_order_id=client_order_id, trading_pair=convert_from_exchange_trading_pair(order["symbol"]), price=Decimal(str(order["price"])), amount=Decimal(str(order["quantity"])), executed_amount=Decimal(str(order["cumQuantity"])), - status=order["status"], + status=order["orderStatus"], order_type=OrderType.LIMIT, - is_buy=True if order["side"].lower() == TradeType.BUY.name.lower() else False, - time=str_date_to_ts(order["createdAt"]), + is_buy=True if order["orderSide"].lower() == TradeType.BUY.name.lower() else False, + time=str_date_to_ts(order["timestamp"]), exchange_order_id=order["id"] ) ) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py b/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py index cf756769c0..275d6c00fb 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py @@ -121,6 +121,9 @@ def update_with_order_update(self, order_update: Dict[str, Any]) -> bool: 'transactTime': '2021-03-23T19:06:51.155520Z' } """ + # Update order execution status + self.last_state = order_update["orderStatus"] + if 'cumulativeQuantity' not in order_update and 'executions' not in order_update: return False @@ -147,7 +150,7 @@ def update_with_order_update(self, order_update: Dict[str, Any]) -> bool: return False self.trade_id_set.add(trade_id) # Set executed amounts - executed_price = Decimal(str(order_update.get("price", "0"))) + executed_price = Decimal(str(order_update.get("averagePrice", order_update.get("price", "0")))) self.executed_amount_base = Decimal(str(order_update["cumulativeQuantity"])) self.executed_amount_quote = executed_price * self.executed_amount_base if self.executed_amount_base <= s_decimal_0: diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py index c3b4fd05de..daeb7213bd 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py @@ -64,7 +64,7 @@ def get_new_client_order_id(is_buy: bool, trading_pair: str) -> str: quote = symbols[1].upper() base_str = f"{base[0]}{base[-1]}" quote_str = f"{quote[0]}{quote[-1]}" - return f"{Constants.HBOT_BROKER_ID}-{side}-{base_str}{quote_str}-{get_tracking_nonce()}" + return f"{Constants.HBOT_BROKER_ID}{side}{base_str}{quote_str}{get_tracking_nonce()}" def retry_sleep_time(try_count: int) -> float: diff --git a/test/connector/exchange/coinzoom/test_coinzoom_exchange.py b/test/connector/exchange/coinzoom/test_coinzoom_exchange.py index 51fcc36b0e..979b4651b6 100644 --- a/test/connector/exchange/coinzoom/test_coinzoom_exchange.py +++ b/test/connector/exchange/coinzoom/test_coinzoom_exchange.py @@ -10,6 +10,7 @@ from typing import List import conf import math +from async_timeout import timeout from hummingbot.core.clock import Clock, ClockMode from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL @@ -88,14 +89,15 @@ def tearDownClass(cls) -> None: async def wait_til_ready(cls, connector = None): if connector is None: connector = cls.connector - while True: - now = time.time() - next_iteration = now // 1.0 + 1 - if connector.ready: - break - else: - await cls._clock.run_til(next_iteration) - await asyncio.sleep(1.0) + async with timeout(90): + while True: + now = time.time() + next_iteration = now // 1.0 + 1 + if connector.ready: + break + else: + await cls._clock.run_til(next_iteration) + await asyncio.sleep(1.0) def setUp(self): self.db_path: str = realpath(join(__file__, "../connector_test.sqlite")) @@ -208,6 +210,7 @@ def test_buy_and_sell(self): def test_limit_makers_unfilled(self): price = self.connector.get_price(self.trading_pair, True) * Decimal("0.8") price = self.connector.quantize_order_price(self.trading_pair, price) + price_quantum = self.connector.get_order_price_quantum(self.trading_pair, price) amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) self.ev_loop.run_until_complete(asyncio.sleep(1)) self.ev_loop.run_until_complete(self.connector._update_balances()) @@ -219,8 +222,7 @@ def test_limit_makers_unfilled(self): self.assertEqual(cl_order_id, order_created_event.order_id) # check available quote balance gets updated, we need to wait a bit for the balance message to arrive taker_fee = self.connector.estimate_fee_pct(False) - quote_amount = ((price * amount)) - quote_amount = ((price * amount) * (Decimal("1") + taker_fee)) + quote_amount = (math.ceil(((price * amount) * (Decimal("1") + taker_fee)) / price_quantum) * price_quantum) expected_quote_bal = quote_bal - quote_amount self.ev_loop.run_until_complete(asyncio.sleep(1)) self.ev_loop.run_until_complete(self.connector._update_balances()) @@ -269,7 +271,7 @@ def test_cancel_all(self): sell_id = self._place_order(False, amount, OrderType.LIMIT, ask_price, 2) self.ev_loop.run_until_complete(asyncio.sleep(1)) - asyncio.ensure_future(self.connector.cancel_all(5)) + asyncio.ensure_future(self.connector.cancel_all(15)) self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) self.ev_loop.run_until_complete(asyncio.sleep(1)) cancel_events = [t for t in self.event_logger.event_log if isinstance(t, OrderCancelledEvent)] From 1945d37b2177ac8b3a5dad4240300dd6d1093b52 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 24 Mar 2021 17:05:41 +0000 Subject: [PATCH 07/23] CoinZoom: Fix Order cancel empty response issue --- hummingbot/connector/exchange/coinzoom/coinzoom_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py index daeb7213bd..599cadc967 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py @@ -81,7 +81,8 @@ async def aiohttp_response_with_errors(request_coroutine): try: parsed_response = await response.json() except Exception: - request_errors = True + if response.status not in [204]: + request_errors = True try: parsed_response = str(await response.read()) if len(parsed_response) > 100: @@ -89,7 +90,7 @@ async def aiohttp_response_with_errors(request_coroutine): except Exception: pass TempFailure = (parsed_response is None or - (response.status not in [200, 201] and "error" not in parsed_response)) + (response.status not in [200, 201, 204] and "error" not in parsed_response)) if TempFailure: parsed_response = response.reason if parsed_response is None else parsed_response request_errors = True From d0087708d122f3acb6c70251f51a53f65c024866 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 24 Mar 2021 17:16:20 +0000 Subject: [PATCH 08/23] CoinZoom: Remove unused code in order book --- .../coinzoom/coinzoom_active_order_tracker.pxd | 3 ++- .../coinzoom/coinzoom_active_order_tracker.pyx | 16 ++++++++-------- .../coinzoom/coinzoom_order_book_message.py | 5 +++++ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd index 752d47418a..881d7862df 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pxd @@ -9,4 +9,5 @@ cdef class CoinzoomActiveOrderTracker: cdef tuple c_convert_diff_message_to_np_arrays(self, object message) cdef tuple c_convert_snapshot_message_to_np_arrays(self, object message) - cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message) + # This method doesn't seem to be used anywhere at all + # cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx index 001f1d4c1c..a7e4fcb815 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_active_order_tracker.pyx @@ -135,14 +135,14 @@ cdef class CoinzoomActiveOrderTracker: return bids, asks - # Is this method actually used? - cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message): - cdef: - double trade_type_value = 1.0 if message.content[4] == "BUY" else 2.0 - list content = message.content - - return np.array([message.timestamp, trade_type_value, float(content[1]), float(content[2])], - dtype="float64") + # This method doesn't seem to be used anywhere at all + # cdef np.ndarray[np.float64_t, ndim=1] c_convert_trade_message_to_np_array(self, object message): + # cdef: + # double trade_type_value = 1.0 if message.content[4] == "BUY" else 2.0 + # list content = message.content + + # return np.array([message.timestamp, trade_type_value, float(content[1]), float(content[2])], + # dtype="float64") def convert_diff_message_to_order_book_row(self, message): np_bids, np_asks = self.c_convert_diff_message_to_np_arrays(message) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py index 4f91267868..d6bc00541d 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_order_book_message.py @@ -47,6 +47,11 @@ def trade_id(self) -> int: def trading_pair(self) -> str: return self.content["trading_pair"] + # The `asks` and `bids` properties are only used in the methods below. + # They are all replaced or unused in this connector: + # OrderBook.restore_from_snapshot_and_diffs + # OrderBookTracker._track_single_book + # MockAPIOrderBookDataSource.get_tracking_pairs @property def asks(self): raise NotImplementedError(Constants.EXCHANGE_NAME + " order book uses active_order_tracker.") From 37e01fe538c62da0720784c8088a0dbc13e9523d Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 24 Mar 2021 17:31:15 +0000 Subject: [PATCH 09/23] CoinZoom: Document "live" exchange URLs --- hummingbot/connector/exchange/coinzoom/coinzoom_constants.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py index d527b4e986..164bf1df76 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py @@ -6,8 +6,11 @@ class Constants: https://api-markets.coinzoom.com/ """ EXCHANGE_NAME = "coinzoom" + # REST_URL = "https://api.coinzoom.com/api/v1/public" REST_URL = "https://api.stage.coinzoom.com/api/v1/public" + # WS_PRIVATE_URL = "wss://api.coinzoom.com/api/v1/public/market/data/stream" WS_PRIVATE_URL = "wss://api.stage.coinzoom.com/api/v1/public/market/data/stream" + # WS_PUBLIC_URL = "wss://api.coinzoom.com/api/v1/public/market/data/stream" WS_PUBLIC_URL = "wss://api.stage.coinzoom.com/api/v1/public/market/data/stream" HBOT_BROKER_ID = "CZ_API_HBOT" From a8c1f6297541ea0a812c4dc7c856fcb6cb37d8d7 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 24 Mar 2021 17:34:38 +0000 Subject: [PATCH 10/23] CoinZoom: Adjust polling interval --- hummingbot/connector/exchange/coinzoom/coinzoom_constants.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py index 164bf1df76..7994650443 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py @@ -52,8 +52,9 @@ class Constants: # Intervals # Only used when nothing is received from WS SHORT_POLL_INTERVAL = 5.0 - # One minute should be fine since we get trades, orders and balances via WS - LONG_POLL_INTERVAL = 60.0 + # CoinZoom poll interval can't be too long since we don't get balances via websockets + LONG_POLL_INTERVAL = 20.0 + # One minute should be fine for order status since we get these via WS UPDATE_ORDER_STATUS_INTERVAL = 60.0 # 10 minute interval to update trading rules, these would likely never change whilst running. INTERVAL_TRADING_RULES = 600 From 8dadaa314119aa6373a86ffd3049085f17125035 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Thu, 25 Mar 2021 01:44:53 +0000 Subject: [PATCH 11/23] CoinZoom: Change Poll interval --- hummingbot/connector/exchange/coinzoom/coinzoom_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py index 7994650443..b58f5bcf82 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py @@ -53,7 +53,7 @@ class Constants: # Only used when nothing is received from WS SHORT_POLL_INTERVAL = 5.0 # CoinZoom poll interval can't be too long since we don't get balances via websockets - LONG_POLL_INTERVAL = 20.0 + LONG_POLL_INTERVAL = 8.0 # One minute should be fine for order status since we get these via WS UPDATE_ORDER_STATUS_INTERVAL = 60.0 # 10 minute interval to update trading rules, these would likely never change whilst running. From ad6907e4d1e98872a2787d5515e8aa7e74c94965 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Fri, 26 Mar 2021 13:05:10 +0000 Subject: [PATCH 12/23] CoinZoom: Change status to yellow --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8c94efb3ea..33dc0ae6ed 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ We created hummingbot to promote **decentralized market-making**: enabling membe | BitMax | bitmax | [BitMax](https://bitmax.io/en/global-digital-asset-platform) | 1 | [API](https://bitmax-exchange.github.io/bitmax-pro-api/#bitmax-pro-api-documentation) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Blocktane | blocktane | [Blocktane](https://blocktane.io/) | 2 | [API](https://blocktane.io/api) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Coinbase Pro | coinbase_pro | [Coinbase Pro](https://pro.coinbase.com/) | * | [API](https://docs.pro.coinbase.com/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | -| CoinZoom | coinzoom | [CoinZoom](https://trade.coinzoom.com/landing) | * | [API](https://api-docs.coinzoom.com/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | +| CoinZoom | coinzoom | [CoinZoom](https://trade.coinzoom.com/landing) | * | [API](https://api-docs.coinzoom.com/) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | | Crypto.com | crypto_com | [Crypto.com](https://crypto.com/exchange) | 2 | [API](https://exchange-docs.crypto.com/#introduction) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | | DyDx | dydx | [dy/dx](https://dydx.exchange/) | 1 | [API](https://docs.dydx.exchange/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Eterbase | eterbase | [Eterbase](https://www.eterbase.com/) | * | [API](https://developers.eterbase.exchange/?version=latest) |![RED](https://via.placeholder.com/15/f03c15/?text=+) | From d3366f46f1203d8ed29aa24427dfaf7f41254fbe Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Fri, 26 Mar 2021 16:57:01 +0000 Subject: [PATCH 13/23] CoinZoom: Debug error on order cancels - TODO: handle this better --- .../connector/exchange/coinzoom/coinzoom_exchange.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py index ab7c69be4f..abe6b9a731 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py @@ -532,12 +532,11 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: raise except CoinzoomAPIError as e: err = e.error_payload.get('error', e.error_payload) - print(f"order cancel error: {err}") + self.logger().error(f"Order Cancel API Error: {err}") # TODO: Still need to handle order cancel errors. - # self._order_not_found_records[order_id] = self._order_not_found_records.get(order_id, 0) + 1 - # if err.get('code') == 20002 and \ - # self._order_not_found_records[order_id] >= self.ORDER_NOT_EXIST_CANCEL_COUNT: - # order_was_cancelled = True + self._order_not_found_records[order_id] = self._order_not_found_records.get(order_id, 0) + 1 + if self._order_not_found_records[order_id] >= self.ORDER_NOT_EXIST_CANCEL_COUNT: + order_was_cancelled = True if order_was_cancelled: self.logger().info(f"Successfully cancelled order {order_id} on {Constants.EXCHANGE_NAME}.") self.stop_tracking_order(order_id) From 28c8356f0abea24427b4f186cbdb28bdc4ce48e4 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Sat, 27 Mar 2021 14:19:59 +0000 Subject: [PATCH 14/23] CoinZoom: Fix error with `test_ws_auth` --- test/connector/exchange/coinzoom/test_coinzoom_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/connector/exchange/coinzoom/test_coinzoom_auth.py b/test/connector/exchange/coinzoom/test_coinzoom_auth.py index bfeeb86cbd..f2573e4460 100644 --- a/test/connector/exchange/coinzoom/test_coinzoom_auth.py +++ b/test/connector/exchange/coinzoom/test_coinzoom_auth.py @@ -35,7 +35,7 @@ async def rest_auth(self) -> Dict[Any, Any]: async def ws_auth(self) -> Dict[Any, Any]: ws = CoinzoomWebsocket(self.auth) await ws.connect() - user_ws_streams = {stream_key: {} for stream_key in Constants.WS_SUB["USER_ORDERS_TRADES"]} + user_ws_streams = {Constants.WS_SUB["USER_ORDERS_TRADES"]: {}} async with timeout(30): await ws.subscribe(user_ws_streams) async for response in ws.on_message(): From b696472758bda4c8c3879dc04f3c1d05d12b55d8 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Sat, 27 Mar 2021 14:34:00 +0000 Subject: [PATCH 15/23] CoinZoom: Document order not found on order cancels Finished the TODO in previous commit, there is no error reported by CoinZoom if an order is not found. --- hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py index abe6b9a731..1a3a16ff4a 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py @@ -533,7 +533,7 @@ async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: except CoinzoomAPIError as e: err = e.error_payload.get('error', e.error_payload) self.logger().error(f"Order Cancel API Error: {err}") - # TODO: Still need to handle order cancel errors. + # CoinZoom doesn't report any error if the order wasn't found so we can only handle API failures here. self._order_not_found_records[order_id] = self._order_not_found_records.get(order_id, 0) + 1 if self._order_not_found_records[order_id] >= self.ORDER_NOT_EXIST_CANCEL_COUNT: order_was_cancelled = True From de815bf33f4e5da5c99f784c97733daa911a66bc Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Sat, 27 Mar 2021 14:41:53 +0000 Subject: [PATCH 16/23] CoinZoom: Enable `originType` order parameter for Broker ID --- .../exchange/coinzoom/coinzoom_exchange.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py index 1a3a16ff4a..4bef239da4 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py @@ -436,8 +436,7 @@ async def _create_order(self, "orderSide": trade_type.name.upper(), "quantity": f"{amount:f}", "price": f"{price:f}", - # Waiting for changes to CoinZoom API for this one. - # "originType": Constants.HBOT_BROKER_ID, + "originType": Constants.HBOT_BROKER_ID, # CoinZoom doesn't support client order id yet # "clientOrderId": order_id, "payFeesWithZoomToken": "true", @@ -680,21 +679,22 @@ def _process_order_message(self, order_msg: Dict[str, Any]): "averagePrice": 56518.7, } """ - if order_msg.get('clientOrderId') is not None: - client_order_id = order_msg["clientOrderId"] - if client_order_id not in self._in_flight_orders: - return - tracked_order = self._in_flight_orders[client_order_id] + # Looks like CoinZoom might support clientOrderId eventually so leaving this here for now. + # if order_msg.get('clientOrderId') is not None: + # client_order_id = order_msg["clientOrderId"] + # if client_order_id not in self._in_flight_orders: + # return + # tracked_order = self._in_flight_orders[client_order_id] + # else: + if "orderId" not in order_msg: + exchange_order_id = str(order_msg["id"]) else: - if "orderId" not in order_msg: - exchange_order_id = str(order_msg["id"]) - else: - exchange_order_id = str(order_msg["orderId"]) - tracked_orders = list(self._in_flight_orders.values()) - track_order = [o for o in tracked_orders if exchange_order_id == o.exchange_order_id] - if not track_order: - return - tracked_order = track_order[0] + exchange_order_id = str(order_msg["orderId"]) + tracked_orders = list(self._in_flight_orders.values()) + track_order = [o for o in tracked_orders if exchange_order_id == o.exchange_order_id] + if not track_order: + return + tracked_order = track_order[0] # Estimate fee order_msg["trade_fee"] = self.estimate_fee_pct(tracked_order.order_type is OrderType.LIMIT_MAKER) From 9b855a033ecdae82b8d5276fdb0755d9a0c72eb7 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 31 Mar 2021 01:01:03 +0100 Subject: [PATCH 17/23] CoinZoom: Switch from stage to production --- .../exchange/coinzoom/coinzoom_constants.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py index b58f5bcf82..a3af9fe30b 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py @@ -6,12 +6,12 @@ class Constants: https://api-markets.coinzoom.com/ """ EXCHANGE_NAME = "coinzoom" - # REST_URL = "https://api.coinzoom.com/api/v1/public" - REST_URL = "https://api.stage.coinzoom.com/api/v1/public" - # WS_PRIVATE_URL = "wss://api.coinzoom.com/api/v1/public/market/data/stream" - WS_PRIVATE_URL = "wss://api.stage.coinzoom.com/api/v1/public/market/data/stream" - # WS_PUBLIC_URL = "wss://api.coinzoom.com/api/v1/public/market/data/stream" - WS_PUBLIC_URL = "wss://api.stage.coinzoom.com/api/v1/public/market/data/stream" + REST_URL = "https://api.coinzoom.com/api/v1/public" + # REST_URL = "https://api.stage.coinzoom.com/api/v1/public" + WS_PRIVATE_URL = "wss://api.coinzoom.com/api/v1/public/market/data/stream" + # WS_PRIVATE_URL = "wss://api.stage.coinzoom.com/api/v1/public/market/data/stream" + WS_PUBLIC_URL = "wss://api.coinzoom.com/api/v1/public/market/data/stream" + # WS_PUBLIC_URL = "wss://api.stage.coinzoom.com/api/v1/public/market/data/stream" HBOT_BROKER_ID = "CZ_API_HBOT" From 38476e060095fb32c0a4c8d7584b81da2c074df9 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 31 Mar 2021 14:00:12 +0100 Subject: [PATCH 18/23] CoinZoom: Fix missing exchange_trade_id on OrderFill --- hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py | 1 + .../connector/exchange/coinzoom/coinzoom_in_flight_order.py | 1 + 2 files changed, 2 insertions(+) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py index 4bef239da4..2536c306bd 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py @@ -729,6 +729,7 @@ async def _trigger_order_fill(self, Decimal(str(update_msg.get("averagePrice", update_msg.get("price", "0")))), tracked_order.executed_amount_base, TradeFee(percent=update_msg["trade_fee"]), + update_msg.get("exchange_trade_id", update_msg.get("id", update_msg.get("orderId"))) ) ) if math.isclose(tracked_order.executed_amount_base, tracked_order.amount) or \ diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py b/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py index 275d6c00fb..61da8fdb0b 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_in_flight_order.py @@ -134,6 +134,7 @@ def update_with_order_update(self, order_update: Dict[str, Any]) -> bool: trade_id = str(trade["timestamp"]) if trade_id not in self.trade_id_set: self.trade_id_set.add(trade_id) + order_update["exchange_trade_id"] = trade.get("id") # Add executed amounts executed_price = Decimal(str(trade.get("lastPrice", "0"))) self.executed_amount_base += Decimal(str(trade["lastQuantity"])) From 2de0012aa74de751e271441be10443a0ded2cb53 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Thu, 1 Apr 2021 16:19:27 +0100 Subject: [PATCH 19/23] CoinZoom: Add logo --- README.md | 2 +- assets/coinzoom_logo.svg | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 assets/coinzoom_logo.svg diff --git a/README.md b/README.md index 504e570869..d3b4b0a8dc 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ We created hummingbot to promote **decentralized market-making**: enabling membe | Bitfinex | bitfinex | [Bitfinex](https://www.bitfinex.com/) | 2 | [API](https://docs.bitfinex.com/docs/introduction) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | | Blocktane | blocktane | [Blocktane](https://blocktane.io/) | 2 | [API](https://blocktane.io/api) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Coinbase Pro | coinbase_pro | [Coinbase Pro](https://pro.coinbase.com/) | * | [API](https://docs.pro.coinbase.com/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | -| CoinZoom | coinzoom | [CoinZoom](https://trade.coinzoom.com/landing) | * | [API](https://api-docs.coinzoom.com/) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | +| CoinZoom | coinzoom | [CoinZoom](https://trade.coinzoom.com/landing) | * | [API](https://api-docs.coinzoom.com/) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | | Crypto.com | crypto_com | [Crypto.com](https://crypto.com/exchange) | 2 | [API](https://exchange-docs.crypto.com/#introduction) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) | | DyDx | dydx | [dy/dx](https://dydx.exchange/) | 1 | [API](https://docs.dydx.exchange/) |![GREEN](https://via.placeholder.com/15/008000/?text=+) | | Eterbase | eterbase | [Eterbase](https://www.eterbase.com/) | * | [API](https://developers.eterbase.exchange/?version=latest) |![RED](https://via.placeholder.com/15/f03c15/?text=+) | diff --git a/assets/coinzoom_logo.svg b/assets/coinzoom_logo.svg new file mode 100644 index 0000000000..8184f907e7 --- /dev/null +++ b/assets/coinzoom_logo.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + From 48298155ae55269ea0438b7526e1a293ba4ac4ab Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Fri, 2 Apr 2021 19:10:44 +0100 Subject: [PATCH 20/23] CoinZoom: Implement Throttler --- .../exchange/coinzoom/coinzoom_exchange.py | 59 ++++++++++--------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py index 2536c306bd..14b92c136d 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py @@ -17,6 +17,7 @@ from hummingbot.core.network_iterator import NetworkStatus from hummingbot.logger import HummingbotLogger from hummingbot.core.clock import Clock +from hummingbot.core.utils.asyncio_throttle import Throttler from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather from hummingbot.connector.trading_rule import TradingRule from hummingbot.core.data_type.cancellation_result import CancellationResult @@ -101,6 +102,7 @@ def __init__(self, self._user_stream_event_listener_task = None self._trading_rules_polling_task = None self._last_poll_timestamp = 0 + self._throttler = Throttler(rate_limit = (8.0, 6)) @property def name(self) -> str: @@ -317,34 +319,35 @@ async def _api_request(self, signature to the request. :returns A response in json format. """ - url = f"{Constants.REST_URL}/{endpoint}" - shared_client = await self._http_client() - # Turn `params` into either GET params or POST body data - qs_params: dict = params if method.upper() == "GET" else None - req_params = ujson.dumps(params) if method.upper() == "POST" and params is not None else None - # Generate auth headers if needed. - headers: dict = {"Content-Type": "application/json"} - if is_auth_required: - headers: dict = self._coinzoom_auth.get_headers() - # Build request coro - response_coro = shared_client.request(method=method.upper(), url=url, headers=headers, - params=qs_params, data=req_params, - timeout=Constants.API_CALL_TIMEOUT) - http_status, parsed_response, request_errors = await aiohttp_response_with_errors(response_coro) - if request_errors or parsed_response is None: - if try_count < Constants.API_MAX_RETRIES: - try_count += 1 - time_sleep = retry_sleep_time(try_count) - self.logger().info(f"Error fetching data from {url}. HTTP status is {http_status}. " - f"Retrying in {time_sleep:.0f}s.") - await asyncio.sleep(time_sleep) - return await self._api_request(method=method, endpoint=endpoint, params=params, - is_auth_required=is_auth_required, try_count=try_count) - else: - raise CoinzoomAPIError({"error": parsed_response, "status": http_status}) - if "error" in parsed_response: - raise CoinzoomAPIError(parsed_response) - return parsed_response + async with self._throttler.weighted_task(request_weight=1): + url = f"{Constants.REST_URL}/{endpoint}" + shared_client = await self._http_client() + # Turn `params` into either GET params or POST body data + qs_params: dict = params if method.upper() == "GET" else None + req_params = ujson.dumps(params) if method.upper() == "POST" and params is not None else None + # Generate auth headers if needed. + headers: dict = {"Content-Type": "application/json"} + if is_auth_required: + headers: dict = self._coinzoom_auth.get_headers() + # Build request coro + response_coro = shared_client.request(method=method.upper(), url=url, headers=headers, + params=qs_params, data=req_params, + timeout=Constants.API_CALL_TIMEOUT) + http_status, parsed_response, request_errors = await aiohttp_response_with_errors(response_coro) + if request_errors or parsed_response is None: + if try_count < Constants.API_MAX_RETRIES: + try_count += 1 + time_sleep = retry_sleep_time(try_count) + self.logger().info(f"Error fetching data from {url}. HTTP status is {http_status}. " + f"Retrying in {time_sleep:.0f}s.") + await asyncio.sleep(time_sleep) + return await self._api_request(method=method, endpoint=endpoint, params=params, + is_auth_required=is_auth_required, try_count=try_count) + else: + raise CoinzoomAPIError({"error": parsed_response, "status": http_status}) + if "error" in parsed_response: + raise CoinzoomAPIError(parsed_response) + return parsed_response def get_order_price_quantum(self, trading_pair: str, price: Decimal): """ From 9cf4d7050ff88108d040b44a6e64dea9a0ffa724 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Fri, 2 Apr 2021 19:11:15 +0100 Subject: [PATCH 21/23] CoinZoom: Add user-agent on unauthenticated requests --- hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py | 2 +- hummingbot/connector/exchange/coinzoom/coinzoom_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py index 14b92c136d..a4ab472965 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py @@ -326,7 +326,7 @@ async def _api_request(self, qs_params: dict = params if method.upper() == "GET" else None req_params = ujson.dumps(params) if method.upper() == "POST" and params is not None else None # Generate auth headers if needed. - headers: dict = {"Content-Type": "application/json"} + headers: dict = {"Content-Type": "application/json", "User-Agent": "hummingbot"} if is_auth_required: headers: dict = self._coinzoom_auth.get_headers() # Build request coro diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py index 599cadc967..498ed56541 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py @@ -105,7 +105,7 @@ async def api_call_with_retries(method, shared_client=None, try_count: int = 0) -> Dict[str, Any]: url = f"{Constants.REST_URL}/{endpoint}" - headers = {"Content-Type": "application/json"} + headers = {"Content-Type": "application/json", "User-Agent": "hummingbot"} http_client = shared_client if shared_client is not None else aiohttp.ClientSession() # Build request coro response_coro = http_client.request(method=method.upper(), url=url, headers=headers, From a154b7957728a924ac99d0dd408efad4a042a7aa Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Fri, 2 Apr 2021 19:13:40 +0100 Subject: [PATCH 22/23] CoinZoom: Add balance check on order update --- hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py index a4ab472965..a4d000accd 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py @@ -705,6 +705,7 @@ def _process_order_message(self, order_msg: Dict[str, Any]): if updated: safe_ensure_future(self._trigger_order_fill(tracked_order, order_msg)) + safe_ensure_future(self._update_balances()) elif tracked_order.is_cancelled: self.logger().info(f"Successfully cancelled order {tracked_order.client_order_id}.") self.stop_tracking_order(tracked_order.client_order_id) From d8a8c6de88a5610bdb3d4ee4f65e22ebcdb3efb3 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Fri, 2 Apr 2021 19:55:32 +0100 Subject: [PATCH 23/23] CoinZoom: Change poll interval and balance update trigger --- hummingbot/connector/exchange/coinzoom/coinzoom_constants.py | 4 ++-- hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py index a3af9fe30b..0cad1cb049 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_constants.py @@ -52,8 +52,8 @@ class Constants: # Intervals # Only used when nothing is received from WS SHORT_POLL_INTERVAL = 5.0 - # CoinZoom poll interval can't be too long since we don't get balances via websockets - LONG_POLL_INTERVAL = 8.0 + # One minute should be fine since we request balance updates on order updates + LONG_POLL_INTERVAL = 60.0 # One minute should be fine for order status since we get these via WS UPDATE_ORDER_STATUS_INTERVAL = 60.0 # 10 minute interval to update trading rules, these would likely never change whilst running. diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py index a4d000accd..65108d7475 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py @@ -702,10 +702,11 @@ def _process_order_message(self, order_msg: Dict[str, Any]): # Estimate fee order_msg["trade_fee"] = self.estimate_fee_pct(tracked_order.order_type is OrderType.LIMIT_MAKER) updated = tracked_order.update_with_order_update(order_msg) + # Call Update balances on every message to catch order create, fill and cancel. + safe_ensure_future(self._update_balances()) if updated: safe_ensure_future(self._trigger_order_fill(tracked_order, order_msg)) - safe_ensure_future(self._update_balances()) elif tracked_order.is_cancelled: self.logger().info(f"Successfully cancelled order {tracked_order.client_order_id}.") self.stop_tracking_order(tracked_order.client_order_id)