From 487833d2938e432745fb2c1f8dd596ce06fff2ff Mon Sep 17 00:00:00 2001 From: vic-en Date: Thu, 4 Feb 2021 12:45:09 +0100 Subject: [PATCH 01/73] (feat) initial part of perpetual protocol connector --- .../derivative/perpetual_finance/__init__.py | 0 .../derivative/perpetual_finance/dummy.pxd | 2 + .../derivative/perpetual_finance/dummy.pyx | 2 + .../perpetual_finance_derivative.py | 557 ++++++++++++++++++ .../perpetual_finance_in_flight_order.py | 58 ++ .../perpetual_finance_utils.py | 34 ++ 6 files changed, 653 insertions(+) create mode 100644 hummingbot/connector/derivative/perpetual_finance/__init__.py create mode 100644 hummingbot/connector/derivative/perpetual_finance/dummy.pxd create mode 100644 hummingbot/connector/derivative/perpetual_finance/dummy.pyx create mode 100644 hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py create mode 100644 hummingbot/connector/derivative/perpetual_finance/perpetual_finance_in_flight_order.py create mode 100644 hummingbot/connector/derivative/perpetual_finance/perpetual_finance_utils.py diff --git a/hummingbot/connector/derivative/perpetual_finance/__init__.py b/hummingbot/connector/derivative/perpetual_finance/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hummingbot/connector/derivative/perpetual_finance/dummy.pxd b/hummingbot/connector/derivative/perpetual_finance/dummy.pxd new file mode 100644 index 0000000000..4b098d6f59 --- /dev/null +++ b/hummingbot/connector/derivative/perpetual_finance/dummy.pxd @@ -0,0 +1,2 @@ +cdef class dummy(): + pass diff --git a/hummingbot/connector/derivative/perpetual_finance/dummy.pyx b/hummingbot/connector/derivative/perpetual_finance/dummy.pyx new file mode 100644 index 0000000000..4b098d6f59 --- /dev/null +++ b/hummingbot/connector/derivative/perpetual_finance/dummy.pyx @@ -0,0 +1,2 @@ +cdef class dummy(): + pass diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py new file mode 100644 index 0000000000..181b277da7 --- /dev/null +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py @@ -0,0 +1,557 @@ +import logging +from decimal import Decimal +import asyncio +import aiohttp +from typing import Dict, Any, List, Optional +import json +import time +import ssl +import copy +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL +from hummingbot.core.utils import async_ttl_cache +from hummingbot.core.network_iterator import NetworkStatus +from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather +from hummingbot.logger import HummingbotLogger +from hummingbot.core.utils.tracking_nonce import get_tracking_nonce +from hummingbot.core.data_type.limit_order import LimitOrder +from hummingbot.core.data_type.cancellation_result import CancellationResult +from hummingbot.core.event.events import ( + MarketEvent, + BuyOrderCreatedEvent, + SellOrderCreatedEvent, + BuyOrderCompletedEvent, + SellOrderCompletedEvent, + MarketOrderFailureEvent, + OrderFilledEvent, + OrderType, + TradeType, + TradeFee, + # PositionSide, PositionMode, + PositionAction +) +from hummingbot.connector.derivative_base import DerivativeBase +from hummingbot.connector.derivative.perpetual_finance.perpetual_finance_in_flight_order import PerpetualFinanceInFlightOrder +from hummingbot.connector.derivative.perpetual_finance.perpetual_finance_utils import convert_to_exchange_trading_pair # convert_from_exchange_trading_pair +from hummingbot.client.settings import GATEAWAY_CA_CERT_PATH, GATEAWAY_CLIENT_CERT_PATH, GATEAWAY_CLIENT_KEY_PATH +from hummingbot.core.utils.eth_gas_station_lookup import get_gas_price +from hummingbot.client.config.global_config_map import global_config_map +# from hummingbot.connector.derivative.position import Position + + +s_logger = None +s_decimal_0 = Decimal("0") +s_decimal_NaN = Decimal("nan") +logging.basicConfig(level=METRICS_LOG_LEVEL) + + +class PerpetualFinanceDerivative(DerivativeBase): + """ + PerpetualFinanceConnector connects with perpetual_finance gateway APIs and provides pricing, user account tracking and trading + functionality. + """ + API_CALL_TIMEOUT = 10.0 + POLL_INTERVAL = 1.0 + UPDATE_BALANCE_INTERVAL = 30.0 + + @classmethod + def logger(cls) -> HummingbotLogger: + global s_logger + if s_logger is None: + s_logger = logging.getLogger(__name__) + return s_logger + + def __init__(self, + trading_pairs: List[str], + wallet_private_key: str, + ethereum_rpc_url: str, + trading_required: bool = True + ): + """ + :param trading_pairs: a list of trading pairs + :param wallet_private_key: a private key for eth wallet + :param ethereum_rpc_url: this is usually infura RPC URL + :param trading_required: Whether actual trading is needed. + """ + super().__init__() + self._trading_pairs = trading_pairs + self._wallet_private_key = wallet_private_key + self._trading_required = trading_required + self._ev_loop = asyncio.get_event_loop() + self._shared_client = None + self._last_poll_timestamp = 0.0 + self._last_balance_poll_timestamp = time.time() + self._in_flight_orders = {} + self._allowances = {} + self._status_polling_task = None + self._auto_approve_task = None + self._real_time_balance_update = False + self._poll_notifier = None + + @property + def name(self): + return "perpetual_finance" + + """@staticmethod + async def fetch_trading_pairs() -> List[str]: + resp = await self._api_request("get", "perpfi/get-pairs") + pairs = resp.get("pairs", []) + if len(pairs) == 0: + await self.load_metadata() + trading_pairs = [] + for pair in pairs: + trading_pairs.append(convert_from_exchange_trading_pair(pair)) + return trading_pairs""" + + @property + def limit_orders(self) -> List[LimitOrder]: + return [ + in_flight_order.to_limit_order() + for in_flight_order in self._in_flight_orders.values() + ] + + async def load_metadata(self): + status = await self._api_request("get", "perpfi/") + loadedMetadata = status["loadedMetadata"] + while (not loadedMetadata): + resp = await self._api_request("get", "perpfi/load-metadata") + loadedMetadata = resp.get("loadedMetadata", False) + return + + async def auto_approve(self): + """ + Automatically approves PerpetualFinance contract as a spender for token in trading pairs. + It first checks if there are any already approved amount (allowance) + """ + self.logger().info("Checking for allowances...") + self._allowances = await self.get_allowances() + for token, amount in self._allowances.items(): + if amount <= s_decimal_0: + amount_approved = await self.approve_perpetual_finance_spender() + if amount_approved > 0: + self._allowances[token] = amount_approved + await asyncio.sleep(2) + else: + break + + async def approve_perpetual_finance_spender(self) -> Decimal: + """ + Approves PerpetualFinance contract as a spender for default USDC token. + """ + resp = await self._api_request("post", "perpfi/approve") + amount_approved = Decimal(str(resp["amount"])) + if amount_approved > 0: + self.logger().info("Approved PerpetualFinance spender contract.") + else: + self.logger().info("PerpetualFinance spender contract approval failed.") + return amount_approved + + async def get_allowances(self) -> Dict[str, Decimal]: + """ + Retrieves allowances for token in trading_pairs + :return: A dictionary of token and its allowance (how much PerpetualFinance can spend). + """ + ret_val = {} + resp = await self._api_request("post", "perpfi/allowances") + for asset, amount in resp["approvals"].items(): + ret_val[asset] = Decimal(str(amount)) + return ret_val + + @async_ttl_cache(ttl=5, maxsize=10) + async def get_quote_price(self, trading_pair: str, is_buy: bool, amount: Decimal) -> Optional[Decimal]: + """ + Retrieves a quote price. + :param trading_pair: The market trading pair + :param is_buy: True for an intention to buy, False for an intention to sell + :param amount: The amount required (in base token unit) + :return: The quote price. + """ + + try: + side = "buy" if is_buy else "sell" + resp = await self._api_request("post", + "perpfi/get-price", + {"side": side, + "pair": convert_to_exchange_trading_pair(trading_pair), + "amount": amount}) + if resp["price"] is not None: + return Decimal(str(resp["price"])) + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().network( + f"Error getting quote price for {trading_pair} {side} order for {amount} amount.", + exc_info=True, + app_warning_msg=str(e) + ) + + async def get_order_price(self, trading_pair: str, is_buy: bool, amount: Decimal) -> Decimal: + """ + This is simply the quote price + """ + return await self.get_quote_price(trading_pair, is_buy, amount) + + def buy(self, trading_pair: str, amount: Decimal, order_type: OrderType, price: Decimal, position_action: PositionAction) -> str: + """ + Buys an amount of base token for a given price (or cheaper). + :param trading_pair: The market trading pair + :param amount: The order amount (in base token unit) + :param order_type: Any order type is fine, not needed for this. + :param price: The maximum price for the order. + :param position_action: Either OPEN or CLOSE position action. + :return: A newly created order id (internal). + """ + return self.place_order(True, trading_pair, amount, price, position_action) + + def sell(self, trading_pair: str, amount: Decimal, order_type: OrderType, price: Decimal, position_action: PositionAction) -> str: + """ + Sells an amount of base token for a given price (or at a higher price). + :param trading_pair: The market trading pair + :param amount: The order amount (in base token unit) + :param order_type: Any order type is fine, not needed for this. + :param price: The minimum price for the order. + :param position_action: Either OPEN or CLOSE position action. + :return: A newly created order id (internal). + """ + return self.place_order(False, trading_pair, amount, price, position_action) + + def place_order(self, is_buy: bool, trading_pair: str, amount: Decimal, price: Decimal, position_action: PositionAction) -> str: + """ + Places an order. + :param is_buy: True for buy order + :param trading_pair: The market trading pair + :param amount: The order amount (in base token unit) + :param price: The minimum price for the order. + :param position_action: Either OPEN or CLOSE position action. + :return: A newly created order id (internal). + """ + side = TradeType.BUY if is_buy else TradeType.SELL + order_id = f"{side.name.lower()}-{trading_pair}-{get_tracking_nonce()}" + safe_ensure_future(self._create_order(side, order_id, trading_pair, amount, price, position_action)) + return order_id + + async def _create_order(self, + trade_type: TradeType, + order_id: str, + trading_pair: str, + amount: Decimal, + price: Decimal, + position_action: PositionAction): + """ + Calls buy or sell API end point to place an order, starts tracking the order and triggers relevant order events. + :param trade_type: BUY or SELL + :param order_id: Internal order id (also called client_order_id) + :param trading_pair: The market to place order + :param amount: The order amount (in base token value) + :param price: The order price + :param position_action: Either OPEN or CLOSE position action. + """ + + amount = self.quantize_order_amount(trading_pair, amount) + price = self.quantize_order_price(trading_pair, price) + base, quote = trading_pair.split("-") + gas_price = get_gas_price() + api_params = {"pair": convert_to_exchange_trading_pair(trading_pair)} + if position_action == PositionAction.OPEN: + api_params.update({"side": 0 if trade_type == TradeType.BUY else 1, + "margin": str(amount / self._leverage), + "leverage": self._leverage, + "minBaseAssetAmount": amount}) + else: + api_params.update({"minimalQuoteAsset": price * amount}) + self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, gas_price) + try: + order_result = await self._api_request("post", f"perpfi/{trade_type.name.lower()}", api_params) + hash = order_result.get("txHash") + tracked_order = self._in_flight_orders.get(order_id) + if tracked_order is not None: + self.logger().info(f"Created {trade_type.name} order {order_id} txHash: {hash} " + f"for {amount} {trading_pair}.") + tracked_order.update_exchange_order_id(hash) + tracked_order.gas_price = gas_price + if hash is not None: + tracked_order.fee_asset = "ETH" + tracked_order.executed_amount_base = amount + tracked_order.executed_amount_quote = amount * price + event_tag = MarketEvent.BuyOrderCreated if trade_type is TradeType.BUY else MarketEvent.SellOrderCreated + event_class = BuyOrderCreatedEvent if trade_type is TradeType.BUY else SellOrderCreatedEvent + self.trigger_event(event_tag, event_class(self.current_timestamp, OrderType.LIMIT, trading_pair, amount, + price, order_id, hash)) + else: + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent(self.current_timestamp, order_id, OrderType.LIMIT)) + except asyncio.CancelledError: + raise + except Exception as e: + self.stop_tracking_order(order_id) + self.logger().network( + f"Error submitting {trade_type.name} order to PerpetualFinance for " + f"{amount} {trading_pair} " + f"{price}.", + exc_info=True, + app_warning_msg=str(e) + ) + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent(self.current_timestamp, order_id, OrderType.LIMIT)) + + def start_tracking_order(self, + order_id: str, + exchange_order_id: str, + trading_pair: str, + trade_type: TradeType, + price: Decimal, + amount: Decimal, + gas_price: Decimal, + leverage: int, + position: str,): + """ + Starts tracking an order by simply adding it into _in_flight_orders dictionary. + """ + self._in_flight_orders[order_id] = PerpetualFinanceInFlightOrder( + client_order_id=order_id, + exchange_order_id=exchange_order_id, + trading_pair=trading_pair, + order_type=OrderType.LIMIT, + trade_type=trade_type, + price=price, + amount=amount, + gas_price=gas_price, + leverage=leverage, + position=position + ) + + def stop_tracking_order(self, order_id: str): + """ + Stops tracking an order by simply removing it from _in_flight_orders dictionary. + """ + if order_id in self._in_flight_orders: + del self._in_flight_orders[order_id] + + async def _update_order_status(self): + """ + Calls REST API to get status update for each in-flight order. + """ + if len(self._in_flight_orders) > 0: + tracked_orders = list(self._in_flight_orders.values()) + + tasks = [] + for tracked_order in tracked_orders: + order_id = await tracked_order.get_exchange_order_id() + tasks.append(self._api_request("post", + "eth/get-receipt", + {"txHash": order_id})) + update_results = await safe_gather(*tasks, return_exceptions=True) + for update_result in update_results: + self.logger().info(f"Polling for order status updates of {len(tasks)} orders.") + if isinstance(update_result, Exception): + raise update_result + if "txHash" not in update_result: + self.logger().info(f"_update_order_status txHash not in resp: {update_result}") + continue + if update_result["confirmed"] is True: + if update_result["receipt"]["status"] == 1: + gas_used = update_result["receipt"]["gasUsed"] + gas_price = tracked_order.gas_price + fee = Decimal(str(gas_used)) * Decimal(str(gas_price)) / Decimal(str(1e9)) + self.trigger_event( + MarketEvent.OrderFilled, + OrderFilledEvent( + self.current_timestamp, + tracked_order.client_order_id, + tracked_order.trading_pair, + tracked_order.trade_type, + tracked_order.order_type, + Decimal(str(tracked_order.price)), + Decimal(str(tracked_order.amount)), + TradeFee(0.0, [(tracked_order.fee_asset, Decimal(str(fee)))]), + exchange_trade_id=order_id + ) + ) + tracked_order.last_state = "FILLED" + self.logger().info(f"The {tracked_order.trade_type.name} order " + f"{tracked_order.client_order_id} has completed " + f"according to order status API.") + event_tag = MarketEvent.BuyOrderCompleted if tracked_order.trade_type is TradeType.BUY \ + else MarketEvent.SellOrderCompleted + event_class = BuyOrderCompletedEvent if tracked_order.trade_type is TradeType.BUY \ + else SellOrderCompletedEvent + self.trigger_event(event_tag, + event_class(self.current_timestamp, + tracked_order.client_order_id, + tracked_order.base_asset, + tracked_order.quote_asset, + tracked_order.fee_asset, + tracked_order.executed_amount_base, + tracked_order.executed_amount_quote, + float(fee), + tracked_order.order_type)) + self.stop_tracking_order(tracked_order.client_order_id) + else: + self.logger().info( + f"The market order {tracked_order.client_order_id} has failed according to order status API. ") + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent( + self.current_timestamp, + tracked_order.client_order_id, + tracked_order.order_type + )) + self.stop_tracking_order(tracked_order.client_order_id) + + def get_taker_order_type(self): + return OrderType.LIMIT + + def get_order_price_quantum(self, trading_pair: str, price: Decimal) -> Decimal: + return Decimal("1e-15") + + def get_order_size_quantum(self, trading_pair: str, order_size: Decimal) -> Decimal: + return Decimal("1e-15") + + @property + def ready(self): + return all(self.status_dict.values()) + + def has_allowances(self) -> bool: + """ + Checks if all tokens have allowance (an amount approved) + """ + return len(self._allowances.values()) == len(self._token_addresses.values()) and \ + all(amount > s_decimal_0 for amount in self._allowances.values()) + + @property + def status_dict(self) -> Dict[str, bool]: + return { + "account_balance": len(self._account_balances) > 0 if self._trading_required else True, + "allowances": self.has_allowances() if self._trading_required else True + } + + async def start_network(self): + if self._trading_required: + self._status_polling_task = safe_ensure_future(self._status_polling_loop()) + self._auto_approve_task = safe_ensure_future(self.auto_approve()) + + async def stop_network(self): + if self._status_polling_task is not None: + self._status_polling_task.cancel() + self._status_polling_task = None + if self._auto_approve_task is not None: + self._auto_approve_task.cancel() + self._auto_approve_task = None + + async def check_network(self) -> NetworkStatus: + try: + response = await self._api_request("get", "api") + if response["status"] != "ok": + raise Exception(f"Error connecting to Gateway API. HTTP status is {response.status}.") + except asyncio.CancelledError: + raise + except Exception: + return NetworkStatus.NOT_CONNECTED + return NetworkStatus.CONNECTED + + def tick(self, timestamp: float): + """ + Is called automatically by the clock for each clock's tick (1 second by default). + It checks if status polling task is due for execution. + """ + if time.time() - self._last_poll_timestamp > self.POLL_INTERVAL: + if self._poll_notifier is not None and not self._poll_notifier.is_set(): + self._poll_notifier.set() + + async def _status_polling_loop(self): + while True: + try: + self._poll_notifier = asyncio.Event() + await self._poll_notifier.wait() + await safe_gather( + self._update_balances(), + self._update_order_status(), + ) + self._last_poll_timestamp = self.current_timestamp + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().error(str(e), exc_info=True) + self.logger().network("Unexpected error while fetching account updates.", + exc_info=True, + app_warning_msg="Could not fetch balances from Gateway API.") + await asyncio.sleep(0.5) + + async def _update_balances(self): + """ + Calls Eth API to update total and available balances. + """ + last_tick = self._last_balance_poll_timestamp + current_tick = self.current_timestamp + if (current_tick - last_tick) > self.UPDATE_BALANCE_INTERVAL: + self._last_balance_poll_timestamp = current_tick + local_asset_names = set(self._account_balances.keys()) + remote_asset_names = set() + resp_json = await self._api_request("post", "perpfi/balances") + for token, bal in resp_json["balances"].items(): + if len(token) > 4: + token = self.get_token(token) + self._account_available_balances[token] = Decimal(str(bal)) + self._account_balances[token] = Decimal(str(bal)) + remote_asset_names.add(token) + + asset_names_to_remove = local_asset_names.difference(remote_asset_names) + for asset_name in asset_names_to_remove: + del self._account_available_balances[asset_name] + del self._account_balances[asset_name] + + self._in_flight_orders_snapshot = {k: copy.copy(v) for k, v in self._in_flight_orders.items()} + self._in_flight_orders_snapshot_timestamp = self.current_timestamp + + async def _http_client(self) -> aiohttp.ClientSession: + """ + :returns Shared client session instance + """ + if self._shared_client is None: + ssl_ctx = ssl.create_default_context(cafile=GATEAWAY_CA_CERT_PATH) + ssl_ctx.load_cert_chain(GATEAWAY_CLIENT_CERT_PATH, GATEAWAY_CLIENT_KEY_PATH) + conn = aiohttp.TCPConnector(ssl_context=ssl_ctx) + self._shared_client = aiohttp.ClientSession(connector=conn) + return self._shared_client + + async def _api_request(self, + method: str, + path_url: str, + params: Dict[str, Any] = {}) -> Dict[str, Any]: + """ + Sends an aiohttp request and waits for a response. + :param method: The HTTP method, e.g. get or post + :param path_url: The path url or the API end point + :param params: A dictionary of required params for the end point + :returns A response in json format. + """ + base_url = f"https://{global_config_map['gateway_api_host'].value}:" \ + f"{global_config_map['gateway_api_port'].value}" + url = f"{base_url}/{path_url}" + client = await self._http_client() + if method == "get": + if len(params) > 0: + response = await client.get(url, params=params) + else: + response = await client.get(url) + elif method == "post": + params["privateKey"] = self._wallet_private_key + if params["privateKey"][:2] != "0x": + params["privateKey"] = "0x" + params["privateKey"] + response = await client.post(url, data=params) + + parsed_response = json.loads(await response.text()) + if response.status != 200: + err_msg = "" + if "error" in parsed_response: + err_msg = f" Message: {parsed_response['error']}" + raise IOError(f"Error fetching data from {url}. HTTP status is {response.status}.{err_msg}") + if "error" in parsed_response: + raise Exception(f"Error: {parsed_response['error']}") + + return parsed_response + + async def cancel_all(self, timeout_seconds: float) -> List[CancellationResult]: + return [] + + @property + def in_flight_orders(self) -> Dict[str, PerpetualFinanceInFlightOrder]: + return self._in_flight_orders diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_in_flight_order.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_in_flight_order.py new file mode 100644 index 0000000000..d7dcc0fc1f --- /dev/null +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_in_flight_order.py @@ -0,0 +1,58 @@ +from decimal import Decimal +from typing import ( + Optional, +) +from hummingbot.core.event.events import ( + OrderType, + TradeType +) +from hummingbot.connector.in_flight_order_base import InFlightOrderBase + + +class PerpetualFinanceInFlightOrder(InFlightOrderBase): + def __init__(self, + client_order_id: str, + exchange_order_id: Optional[str], + trading_pair: str, + order_type: OrderType, + trade_type: TradeType, + price: Decimal, + amount: Decimal, + gas_price: Decimal, + leverage: int, + position: str, + initial_state: str = "OPEN"): + super().__init__( + client_order_id, + exchange_order_id, + trading_pair, + order_type, + trade_type, + price, + amount, + initial_state, + ) + self.trade_id_set = set() + self._gas_price = gas_price + self.leverage = leverage + self.position = position + + @property + def is_done(self) -> bool: + return self.last_state in {"FILLED", "CANCELED", "REJECTED", "EXPIRED"} + + @property + def is_failure(self) -> bool: + return self.last_state in {"REJECTED"} + + @property + def is_cancelled(self) -> bool: + return self.last_state in {"CANCELED", "EXPIRED"} + + @property + def gas_price(self) -> Decimal: + return self._gas_price + + @gas_price.setter + def gas_price(self, gas_price) -> Decimal: + self._gas_price = gas_price diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_utils.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_utils.py new file mode 100644 index 0000000000..665a0a34e1 --- /dev/null +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_utils.py @@ -0,0 +1,34 @@ +import re +from typing import Optional, Tuple + +CENTRALIZED = False +EXAMPLE_PAIR = "ETH-USDC" +DEFAULT_FEES = [0.1, 0.1] + +USE_ETHEREUM_WALLET = True +FEE_TYPE = "FlatFee" +FEE_TOKEN = "XDAI" + +USE_ETH_GAS_LOOKUP = False + +QUOTE = re.compile(r"^(\w+)(USDC|USDT)$") + + +# Helper Functions --- +def split_trading_pair(trading_pair: str) -> Optional[Tuple[str, str]]: + try: + m = QUOTE.match(trading_pair) + return m.group(1), m.group(2) + except Exception as e: + raise e + + +def convert_from_exchange_trading_pair(exchange_trading_pair: str) -> Optional[str]: + if split_trading_pair(exchange_trading_pair) is None: + return None + base_asset, quote_asset = split_trading_pair(exchange_trading_pair) + return f"{base_asset}-{quote_asset}" + + +def convert_to_exchange_trading_pair(hb_trading_pair: str) -> str: + return hb_trading_pair.replace("-", "") From 16e03b3853c83ce087ac1a682b9eac24d95b60e9 Mon Sep 17 00:00:00 2001 From: vic-en Date: Mon, 8 Feb 2021 12:55:16 +0100 Subject: [PATCH 02/73] (refactor) refactor existing derivative functions and populate DerivativeBase class --- .../binance_perpetual_derivative.py | 35 ++++++---------- hummingbot/connector/derivative_base.py | 42 +++++++++++++++++++ .../perpetual_market_making.pyx | 2 +- 3 files changed, 55 insertions(+), 24 deletions(-) diff --git a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py index 910f88b93d..5670407a0c 100644 --- a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py +++ b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py @@ -131,10 +131,6 @@ def __init__(self, self._trading_rules_polling_task = None self._last_poll_timestamp = 0 self._throttler = Throttler((10.0, 1.0)) - self._funding_rate = 0 - self._account_positions = {} - self._position_mode = None - self._leverage = 1 @property def name(self) -> str: @@ -408,10 +404,6 @@ async def execute_cancel(self, trading_pair: str, client_order_id: str): OrderCancelledEvent(self.current_timestamp, client_order_id)) return response - # TODO: Implement - async def close_position(self, trading_pair: str): - pass - def quantize_order_amount(self, trading_pair: str, amount: object, price: object = Decimal(0)): trading_rule: TradingRule = self._trading_rules[trading_pair] # current_price: object = self.get_price(trading_pair, False) @@ -883,7 +875,7 @@ async def _update_order_status(self): order_type)) self.stop_tracking_order(client_order_id) - async def _set_margin(self, trading_pair: str, leverage: int = 1): + async def _set_leverage(self, trading_pair: str, leverage: int = 1): params = { "symbol": convert_to_exchange_trading_pair(trading_pair), "leverage": leverage @@ -902,23 +894,16 @@ async def _set_margin(self, trading_pair: str, leverage: int = 1): self.logger().error("Unable to set leverage.") return leverage - def set_margin(self, trading_pair: str, leverage: int = 1): - safe_ensure_future(self._set_margin(trading_pair, leverage)) - - """ - async def get_position_pnl(self, trading_pair: str): - await self._update_positions() - return self._account_positions.get(trading_pair) - """ + def set_leverage(self, trading_pair: str, leverage: int = 1): + safe_ensure_future(self._set_leverage(trading_pair, leverage)) - async def _get_funding_rate(self, trading_pair): - # TODO: Note --- the "premiumIndex" endpoint can get markPrice, indexPrice, and nextFundingTime as well + async def _get_funding_info(self, trading_pair): prem_index = await self.request("/fapi/v1/premiumIndex", params={"symbol": convert_to_exchange_trading_pair(trading_pair)}) - self._funding_rate = Decimal(prem_index.get("lastFundingRate", "0")) + self._funding_info = Decimal(prem_index.get("lastFundingRate", "0")) - def get_funding_rate(self, trading_pair): - safe_ensure_future(self._get_funding_rate(trading_pair)) - return self._funding_rate + def get_funding_info(self, trading_pair): + safe_ensure_future(self._get_funding_info(trading_pair)) + return self._funding_info async def _set_position_mode(self, position_mode: PositionMode): initial_mode = await self._get_position_mode() @@ -941,6 +926,7 @@ async def _set_position_mode(self, position_mode: PositionMode): self.logger().info(f"Using {position_mode.name} position mode.") async def _get_position_mode(self): + # To-do: ensure there's no active order or contract before changing position mode if self._position_mode is None: mode = await self.request( path="/fapi/v1/positionSide/dual", @@ -955,6 +941,9 @@ async def _get_position_mode(self): def set_position_mode(self, position_mode: PositionMode): safe_ensure_future(self._set_position_mode(position_mode)) + def supported_position_modes(self): + return [PositionMode.ONEWAY, PositionMode.HEDGE] + async def request(self, path: str, params: Dict[str, Any] = {}, method: MethodType = MethodType.GET, add_timestamp: bool = False, is_signed: bool = False, request_weight: int = 1, return_err: bool = False): async with self._throttler.weighted_task(request_weight): diff --git a/hummingbot/connector/derivative_base.py b/hummingbot/connector/derivative_base.py index 668d568156..59e4d7b64c 100644 --- a/hummingbot/connector/derivative_base.py +++ b/hummingbot/connector/derivative_base.py @@ -1,5 +1,6 @@ from decimal import Decimal from hummingbot.connector.exchange_base import ExchangeBase +from hummingbot.core.event.events import PositionMode NaN = float("nan") @@ -14,3 +15,44 @@ class DerivativeBase(ExchangeBase): def __init__(self): super().__init__() + self._funding_info = {} + self._account_positions = {} + self._position_mode = None + self._leverage = 1 + + def set_position_mode(self, position_mode: PositionMode): + """ + Should set the _position_mode parameter. i.e self._position_mode = position_mode + This should also be overwritten if the derivative exchange requires interraction to set mode, + in addition to setting the _position_mode object. + :param position_mode: ONEWAY or HEDGE position mode + """ + self._position_mode = position_mode + return + + def set_leverage(self, trading_pair: str, leverage: int = 1): + """ + Should set the _leverage parameter. i.e self._leverage = leverage + This should also be overwritten if the derivative exchange requires interraction to set leverage, + in addition to setting the _leverage object. + :param _leverage: leverage to be used + """ + self._leverage = leverage + return + + def supported_position_modes(self): + """ + returns a list containing the modes supported by the derivative + ONEWAY and/or HEDGE modes + """ + return [PositionMode.ONEWAY] + + def get_funding_info(self, trading_pair): + """ + return a dictionary containing: + "indexPrice": (i.e "21.169488483519444444") + "markPrice": price used for both pnl on most derivatives (i.e "21.210103847902463671") + "nextFundingTime": next funding time in unix timestamp (i.e "1612780270") + "rate": next funding rate as a decimal and not percentage (i.e 0.00007994084744229488) + """ + raise NotImplementedError diff --git a/hummingbot/strategy/perpetual_market_making/perpetual_market_making.pyx b/hummingbot/strategy/perpetual_market_making/perpetual_market_making.pyx index 5eea14483d..5f7b879d21 100644 --- a/hummingbot/strategy/perpetual_market_making/perpetual_market_making.pyx +++ b/hummingbot/strategy/perpetual_market_making/perpetual_market_making.pyx @@ -555,7 +555,7 @@ cdef class PerpetualMarketMakingStrategy(StrategyBase): cdef c_apply_initial_settings(self, str trading_pair, object position, int64_t leverage): cdef: ExchangeBase market = self._market_info.market - market.set_margin(trading_pair, leverage) + market.set_leverage(trading_pair, leverage) market.set_position_mode(position) cdef c_tick(self, double timestamp): From cf3de6740e7501181cfc01f6bc1d893d2282280b Mon Sep 17 00:00:00 2001 From: vic-en Date: Wed, 10 Feb 2021 15:13:23 +0100 Subject: [PATCH 03/73] (feat) add derivative features and fix fetch_trading_pair --- .../binance_perpetual_derivative.py | 4 - .../perpetual_finance_derivative.py | 105 ++++++++++++------ .../perpetual_finance_in_flight_order.py | 20 ++-- 3 files changed, 86 insertions(+), 43 deletions(-) diff --git a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py index 5670407a0c..b3b132693a 100644 --- a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py +++ b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py @@ -711,11 +711,7 @@ async def _update_balances(self): del self._account_available_balances[asset_name] del self._account_balances[asset_name] - # TODO: Note --- Data Structure Assumes One-way Position Mode [not hedge position mode] (see Binance Futures Docs) - # Note --- Hedge Mode allows for Both Long and Short Positions on a trading pair async def _update_positions(self): - # local_position_names = set(self._account_positions.keys()) - # remote_position_names = set() positions = await self.request(path="/fapi/v2/positionRisk", add_timestamp=True, is_signed=True) for position in positions: trading_pair = position.get("symbol") diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py index 181b277da7..03626b6dd0 100644 --- a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py @@ -13,6 +13,7 @@ from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather from hummingbot.logger import HummingbotLogger from hummingbot.core.utils.tracking_nonce import get_tracking_nonce +from hummingbot.core.utils.estimate_fee import estimate_fee from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.cancellation_result import CancellationResult from hummingbot.core.event.events import ( @@ -25,17 +26,15 @@ OrderFilledEvent, OrderType, TradeType, - TradeFee, - # PositionSide, PositionMode, + PositionSide, PositionAction ) from hummingbot.connector.derivative_base import DerivativeBase from hummingbot.connector.derivative.perpetual_finance.perpetual_finance_in_flight_order import PerpetualFinanceInFlightOrder -from hummingbot.connector.derivative.perpetual_finance.perpetual_finance_utils import convert_to_exchange_trading_pair # convert_from_exchange_trading_pair +from hummingbot.connector.derivative.perpetual_finance.perpetual_finance_utils import convert_to_exchange_trading_pair, convert_from_exchange_trading_pair from hummingbot.client.settings import GATEAWAY_CA_CERT_PATH, GATEAWAY_CLIENT_CERT_PATH, GATEAWAY_CLIENT_KEY_PATH -from hummingbot.core.utils.eth_gas_station_lookup import get_gas_price from hummingbot.client.config.global_config_map import global_config_map -# from hummingbot.connector.derivative.position import Position +from hummingbot.connector.derivative.position import Position s_logger = None @@ -51,7 +50,7 @@ class PerpetualFinanceDerivative(DerivativeBase): """ API_CALL_TIMEOUT = 10.0 POLL_INTERVAL = 1.0 - UPDATE_BALANCE_INTERVAL = 30.0 + UPDATE_BALANCE_INTERVAL = 5.0 @classmethod def logger(cls) -> HummingbotLogger: @@ -91,16 +90,38 @@ def __init__(self, def name(self): return "perpetual_finance" - """@staticmethod + @staticmethod async def fetch_trading_pairs() -> List[str]: - resp = await self._api_request("get", "perpfi/get-pairs") - pairs = resp.get("pairs", []) - if len(pairs) == 0: - await self.load_metadata() + ssl_ctx = ssl.create_default_context(cafile=GATEAWAY_CA_CERT_PATH) + ssl_ctx.load_cert_chain(GATEAWAY_CLIENT_CERT_PATH, GATEAWAY_CLIENT_KEY_PATH) + conn = aiohttp.TCPConnector(ssl_context=ssl_ctx) + client = aiohttp.ClientSession(connector=conn) + + base_url = f"https://{global_config_map['gateway_api_host'].value}:" \ + f"{global_config_map['gateway_api_port'].value}/perpfi/" + response = await client.get(base_url + "pairs") + parsed_response = json.loads(await response.text()) + if response.status != 200: + err_msg = "" + if "error" in parsed_response: + err_msg = f" Message: {parsed_response['error']}" + raise IOError(f"Error fetching pairs from gateway. HTTP status is {response.status}.{err_msg}") + pairs = parsed_response.get("pairs", []) + if "error" in parsed_response or len(pairs) == 0: + raise Exception(f"Error: {parsed_response['error']}") + else: + status = await client.get(base_url) + status = json.loads(await status.text()) + loadedMetadata = status["loadedMetadata"] + while (not loadedMetadata): + resp = await client.get(base_url + "load-metadata") + resp = json.loads(await resp.text()) + loadedMetadata = resp.get("loadedMetadata", False) + return PerpetualFinanceDerivative.fetch_trading_pairs() trading_pairs = [] for pair in pairs: trading_pairs.append(convert_from_exchange_trading_pair(pair)) - return trading_pairs""" + return trading_pairs @property def limit_orders(self) -> List[LimitOrder]: @@ -169,7 +190,7 @@ async def get_quote_price(self, trading_pair: str, is_buy: bool, amount: Decimal try: side = "buy" if is_buy else "sell" resp = await self._api_request("post", - "perpfi/get-price", + "perpfi/price", {"side": side, "pair": convert_to_exchange_trading_pair(trading_pair), "amount": amount}) @@ -249,7 +270,6 @@ async def _create_order(self, amount = self.quantize_order_amount(trading_pair, amount) price = self.quantize_order_price(trading_pair, price) base, quote = trading_pair.split("-") - gas_price = get_gas_price() api_params = {"pair": convert_to_exchange_trading_pair(trading_pair)} if position_action == PositionAction.OPEN: api_params.update({"side": 0 if trade_type == TradeType.BUY else 1, @@ -258,18 +278,17 @@ async def _create_order(self, "minBaseAssetAmount": amount}) else: api_params.update({"minimalQuoteAsset": price * amount}) - self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, gas_price) + self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, self._leverage, position_action.name) try: - order_result = await self._api_request("post", f"perpfi/{trade_type.name.lower()}", api_params) + order_result = await self._api_request("post", f"perpfi/{position_action.name.lower()}", api_params) hash = order_result.get("txHash") tracked_order = self._in_flight_orders.get(order_id) if tracked_order is not None: self.logger().info(f"Created {trade_type.name} order {order_id} txHash: {hash} " f"for {amount} {trading_pair}.") tracked_order.update_exchange_order_id(hash) - tracked_order.gas_price = gas_price if hash is not None: - tracked_order.fee_asset = "ETH" + tracked_order.fee_asset = "XDAI" tracked_order.executed_amount_base = amount tracked_order.executed_amount_quote = amount * price event_tag = MarketEvent.BuyOrderCreated if trade_type is TradeType.BUY else MarketEvent.SellOrderCreated @@ -300,7 +319,6 @@ def start_tracking_order(self, trade_type: TradeType, price: Decimal, amount: Decimal, - gas_price: Decimal, leverage: int, position: str,): """ @@ -314,7 +332,6 @@ def start_tracking_order(self, trade_type=trade_type, price=price, amount=amount, - gas_price=gas_price, leverage=leverage, position=position ) @@ -337,7 +354,7 @@ async def _update_order_status(self): for tracked_order in tracked_orders: order_id = await tracked_order.get_exchange_order_id() tasks.append(self._api_request("post", - "eth/get-receipt", + "perpfi/receipt", {"txHash": order_id})) update_results = await safe_gather(*tasks, return_exceptions=True) for update_result in update_results: @@ -349,9 +366,8 @@ async def _update_order_status(self): continue if update_result["confirmed"] is True: if update_result["receipt"]["status"] == 1: - gas_used = update_result["receipt"]["gasUsed"] - gas_price = tracked_order.gas_price - fee = Decimal(str(gas_used)) * Decimal(str(gas_price)) / Decimal(str(1e9)) + fee = estimate_fee("perpetual_finance", False) + fee.flat_fees = [(tracked_order.fee_asset, Decimal(str(update_result["receipt"]["gasUsed"])))] self.trigger_event( MarketEvent.OrderFilled, OrderFilledEvent( @@ -362,7 +378,7 @@ async def _update_order_status(self): tracked_order.order_type, Decimal(str(tracked_order.price)), Decimal(str(tracked_order.amount)), - TradeFee(0.0, [(tracked_order.fee_asset, Decimal(str(fee)))]), + fee, exchange_trade_id=order_id ) ) @@ -413,8 +429,7 @@ def has_allowances(self) -> bool: """ Checks if all tokens have allowance (an amount approved) """ - return len(self._allowances.values()) == len(self._token_addresses.values()) and \ - all(amount > s_decimal_0 for amount in self._allowances.values()) + return all(amount > s_decimal_0 for amount in self._allowances.values()) @property def status_dict(self) -> Dict[str, bool]: @@ -485,12 +500,12 @@ async def _update_balances(self): self._last_balance_poll_timestamp = current_tick local_asset_names = set(self._account_balances.keys()) remote_asset_names = set() - resp_json = await self._api_request("post", "perpfi/balances") - for token, bal in resp_json["balances"].items(): - if len(token) > 4: - token = self.get_token(token) + balances_resp = await self._api_request("post", "perpfi/balances") + margin_resp = await self._api_request("post", "perpfi/margin") + for token, bal in balances_resp["balances"].items(): self._account_available_balances[token] = Decimal(str(bal)) - self._account_balances[token] = Decimal(str(bal)) + self._account_balances[token] = Decimal(str(bal)) + Decimal(str(margin_resp["margin"])) if token == "USDC" \ + else Decimal(str(bal)) remote_asset_names.add(token) asset_names_to_remove = local_asset_names.difference(remote_asset_names) @@ -501,6 +516,32 @@ async def _update_balances(self): self._in_flight_orders_snapshot = {k: copy.copy(v) for k, v in self._in_flight_orders.items()} self._in_flight_orders_snapshot_timestamp = self.current_timestamp + async def _update_positions(self): + tasks = [] + for pair in self._trading_pairs: + tasks.append(self._api_request("post", + "perpfi/position", + {"pair": convert_to_exchange_trading_pair(pair)})) + positions = await safe_gather(*tasks, return_exceptions=True) + for trading_pair, position in zip(self._trading_pairs, positions.get("position", {})): + position_side = PositionSide.LONG if position.get("size", 0) > 0 else PositionSide.SHORT + unrealized_pnl = Decimal(position.get("pnl")) + entry_price = Decimal(position.get("entryPrice")) + amount = Decimal(position.get("size")) + leverage = self._leverage + if amount != 0: + self._account_positions[trading_pair + position_side.name] = Position( + trading_pair=trading_pair, + position_side=position_side, + unrealized_pnl=unrealized_pnl, + entry_price=entry_price, + amount=amount, + leverage=leverage + ) + else: + if (trading_pair + position_side.name) in self._account_positions: + del self._account_positions[trading_pair + position_side.name] + async def _http_client(self) -> aiohttp.ClientSession: """ :returns Shared client session instance diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_in_flight_order.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_in_flight_order.py index d7dcc0fc1f..3dbaad3128 100644 --- a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_in_flight_order.py +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_in_flight_order.py @@ -18,7 +18,6 @@ def __init__(self, trade_type: TradeType, price: Decimal, amount: Decimal, - gas_price: Decimal, leverage: int, position: str, initial_state: str = "OPEN"): @@ -33,7 +32,6 @@ def __init__(self, initial_state, ) self.trade_id_set = set() - self._gas_price = gas_price self.leverage = leverage self.position = position @@ -50,9 +48,17 @@ def is_cancelled(self) -> bool: return self.last_state in {"CANCELED", "EXPIRED"} @property - def gas_price(self) -> Decimal: - return self._gas_price + def leverage(self) -> Decimal: + return self.leverage - @gas_price.setter - def gas_price(self, gas_price) -> Decimal: - self._gas_price = gas_price + @leverage.setter + def leverage(self, leverage) -> Decimal: + self.leverage = leverage + + @property + def position(self) -> Decimal: + return self.position + + @position.setter + def position(self, position) -> Decimal: + self.position = position From 6841127af9a72a73991ea4396a712e32ac88a06b Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Sun, 14 Feb 2021 13:50:25 +0800 Subject: [PATCH 04/73] (add) probit orderbook tracker [WIP] --- .../connector/exchange/probit/__init__.py | 0 .../probit_api_order_book_data_source.py | 204 ++++++++++++++++++ .../exchange/probit/probit_constants.py | 13 ++ .../exchange/probit/probit_order_book.py | 146 +++++++++++++ .../probit/probit_order_book_message.py | 0 .../probit/probit_order_book_tracker.py | 0 .../probit/probit_order_book_tracker_entry.py | 0 .../connector/exchange/probit/probit_utils.py | 18 ++ 8 files changed, 381 insertions(+) create mode 100644 hummingbot/connector/exchange/probit/__init__.py create mode 100644 hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py create mode 100644 hummingbot/connector/exchange/probit/probit_constants.py create mode 100644 hummingbot/connector/exchange/probit/probit_order_book.py create mode 100644 hummingbot/connector/exchange/probit/probit_order_book_message.py create mode 100644 hummingbot/connector/exchange/probit/probit_order_book_tracker.py create mode 100644 hummingbot/connector/exchange/probit/probit_order_book_tracker_entry.py create mode 100644 hummingbot/connector/exchange/probit/probit_utils.py diff --git a/hummingbot/connector/exchange/probit/__init__.py b/hummingbot/connector/exchange/probit/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py b/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py new file mode 100644 index 0000000000..0bdb8ed737 --- /dev/null +++ b/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python +import asyncio +import logging +import time +import aiohttp +import pandas as pd +import hummingbot.connector.exchange.probit.probit_constants as constants + +from typing import Optional, List, Dict, Any +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_message import OrderBookMessage +from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource +from hummingbot.logger import HummingbotLogger +from . import probit_utils +from .probit_order_book import ProbitOrderBook +from .probit_websocket import ProbitWebsocket +from .probit_utils import ms_timestamp_to_s + + +class ProbitAPIOrderBookDataSource(OrderBookTrackerDataSource): + MAX_RETRIES = 20 + MESSAGE_TIMEOUT = 30.0 + SNAPSHOT_TIMEOUT = 10.0 + + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, trading_pairs: List[str] = None): + super().__init__(trading_pairs) + self._trading_pairs: List[str] = trading_pairs + self._snapshot_msg: Dict[str, any] = {} + + @classmethod + async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, float]: + result = {} + async with aiohttp.ClientSession() as client: + async with client.get(f"{constants.TICKER_PATH_URL}") as response: + if response.status == 200: + resp_json = await response.json() + if "data" in resp_json: + for trading_pair in resp_json["data"]: + result[trading_pair["market_id"]] = trading_pair["last"] + return result + + @staticmethod + async def fetch_trading_pairs() -> List[str]: + async with aiohttp.ClientSession() as client: + async with client.get(f"{constants.MARKETS_PATH_URL}") as response: + if response.status == 200: + resp_json: Dict[str, Any] = await response.json() + return [market["market_id"] for market in resp_json["data"]] + return [] + + @staticmethod + async def get_order_book_data(trading_pair: str) -> Dict[str, any]: + """ + Get whole orderbook + """ + async with aiohttp.ClientSession() as client: + async with client.get(url=f"{constants.ORDER_BOOK_PATH_URL}", + params={"market_id": trading_pair}) as response: + if response.status != 200: + raise IOError( + f"Error fetching OrderBook for {trading_pair} at {constants.ORDER_BOOK_PATH_URL}. " + f"HTTP {response.status}. Response: {await response.json()}" + ) + return await response.json() + + async def get_new_order_book(self, trading_pair: str) -> OrderBook: + snapshot: Dict[str, Any] = await self.get_order_book_data(trading_pair) + snapshot_timestamp: float = time.time() + snapshot_msg: OrderBookMessage = ProbitOrderBook.snapshot_message_from_exchange( + snapshot, + snapshot_timestamp, + metadata={"trading_pair": trading_pair} + ) + order_book = self.order_book_create_function() + bids, asks = probit_utils.convert_snapshot_message_to_order_book_row(snapshot_msg) + order_book.apply_snapshot(bids, asks, snapshot_msg.update_id) + return order_book + + async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for trades using websocket trade channel + """ + while True: + try: + ws = ProbitWebsocket() + await ws.connect() + + await ws.subscribe(list(map( + lambda pair: f"trade.{probit_utils.convert_to_exchange_trading_pair(pair)}", + self._trading_pairs + ))) + + async for response in ws.on_message(): + if response.get("result") is None: + continue + + for trade in response["result"]["data"]: + trade: Dict[Any] = trade + trade_timestamp: int = ms_timestamp_to_s(trade["t"]) + trade_msg: OrderBookMessage = ProbitOrderBook.trade_message_from_exchange( + trade, + trade_timestamp, + metadata={"trading_pair": probit_utils.convert_from_exchange_trading_pair(trade["i"])} + ) + output.put_nowait(trade_msg) + + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error.", exc_info=True) + await asyncio.sleep(5.0) + finally: + await ws.disconnect() + + async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for orderbook diffs using websocket book channel + """ + while True: + try: + ws = ProbitWebsocket() + await ws.connect() + + await ws.subscribe(list(map( + lambda pair: f"book.{probit_utils.convert_to_exchange_trading_pair(pair)}.150", + self._trading_pairs + ))) + + async for response in ws.on_message(): + if response.get("result") is None: + continue + + order_book_data = response["result"]["data"][0] + timestamp: int = ms_timestamp_to_s(order_book_data["t"]) + # data in this channel is not order book diff but the entire order book (up to depth 150). + # so we need to convert it into a order book snapshot. + # Crypto.com does not offer order book diff ws updates. + orderbook_msg: OrderBookMessage = ProbitOrderBook.snapshot_message_from_exchange( + order_book_data, + timestamp, + metadata={"trading_pair": probit_utils.convert_from_exchange_trading_pair( + response["result"]["instrument_name"])} + ) + output.put_nowait(orderbook_msg) + + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unexpected error with WebSocket connection.", + exc_info=True, + app_warning_msg="Unexpected error with WebSocket connection. Retrying in 30 seconds. " + "Check network connection." + ) + await asyncio.sleep(30.0) + finally: + await ws.disconnect() + + async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + """ + Listen for orderbook snapshots by fetching orderbook + """ + while True: + try: + for trading_pair in self._trading_pairs: + try: + snapshot: Dict[str, any] = await self.get_order_book_data(trading_pair) + snapshot_timestamp: int = ms_timestamp_to_s(snapshot["t"]) + snapshot_msg: OrderBookMessage = ProbitOrderBook.snapshot_message_from_exchange( + snapshot, + snapshot_timestamp, + metadata={"trading_pair": trading_pair} + ) + output.put_nowait(snapshot_msg) + self.logger().debug(f"Saved order book snapshot for {trading_pair}") + # Be careful not to go above API rate limits. + await asyncio.sleep(5.0) + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unexpected error with WebSocket connection.", + exc_info=True, + app_warning_msg="Unexpected error with WebSocket connection. Retrying in 5 seconds. " + "Check network connection." + ) + await asyncio.sleep(5.0) + this_hour: pd.Timestamp = pd.Timestamp.utcnow().replace(minute=0, second=0, microsecond=0) + next_hour: pd.Timestamp = this_hour + pd.Timedelta(hours=1) + delta: float = next_hour.timestamp() - time.time() + await asyncio.sleep(delta) + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error.", exc_info=True) + await asyncio.sleep(5.0) diff --git a/hummingbot/connector/exchange/probit/probit_constants.py b/hummingbot/connector/exchange/probit/probit_constants.py new file mode 100644 index 0000000000..5486551910 --- /dev/null +++ b/hummingbot/connector/exchange/probit/probit_constants.py @@ -0,0 +1,13 @@ +# A single source of truth for constant variables related to the exchange + +EXCHANGE_NAME = "probit" + +REST_URL = "https://api.probit.com/api/exchange/" +WSS_URL = "wss://api.probit.com/api/exchange/v1/ws" + +API_VERSON = "v1" + +TICKER_PATH_URL = f"{REST_URL+API_VERSON}/ticker" +MARKETS_PATH_URL = f"{REST_URL+API_VERSON}/market" +ORDER_BOOK_PATH_URL = f"{REST_URL+API_VERSON}/order_book" +NEW_ORDER_PATH_URL = f"{REST_URL+API_VERSON}/new_order" diff --git a/hummingbot/connector/exchange/probit/probit_order_book.py b/hummingbot/connector/exchange/probit/probit_order_book.py new file mode 100644 index 0000000000..7217094899 --- /dev/null +++ b/hummingbot/connector/exchange/probit/probit_order_book.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python + +import logging +import hummingbot.connector.exchange.probit.probit_constants as constants + +from sqlalchemy.engine import RowProxy +from typing import ( + Optional, + Dict, + List, Any) +from hummingbot.logger import HummingbotLogger +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_message import ( + OrderBookMessage, OrderBookMessageType +) +from hummingbot.connector.exchange.probit.probit_order_book_message import ProbitOrderBookMessage + +_logger = None + + +class ProbitOrderBook(OrderBook): + @classmethod + def logger(cls) -> HummingbotLogger: + global _logger + if _logger is None: + _logger = logging.getLogger(__name__) + return _logger + + @classmethod + def snapshot_message_from_exchange(cls, + msg: Dict[str, any], + timestamp: float, + metadata: Optional[Dict] = None): + """ + Convert json snapshot data into standard OrderBookMessage format + :param msg: json snapshot data from live web socket stream + :param timestamp: timestamp attached to incoming data + :return: ProbitOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + return ProbitOrderBookMessage( + message_type=OrderBookMessageType.SNAPSHOT, + content=msg, + timestamp=timestamp + ) + + @classmethod + def snapshot_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of snapshot data into standard OrderBookMessage format + :param record: a row of snapshot data from the database + :return: ProbitOrderBookMessage + """ + return ProbitOrderBookMessage( + message_type=OrderBookMessageType.SNAPSHOT, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def diff_message_from_exchange(cls, + msg: Dict[str, any], + timestamp: Optional[float] = None, + metadata: Optional[Dict] = None): + """ + Convert json diff data into standard OrderBookMessage format + :param msg: json diff data from live web socket stream + :param timestamp: timestamp attached to incoming data + :return: ProbitOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + return ProbitOrderBookMessage( + message_type=OrderBookMessageType.DIFF, + content=msg, + timestamp=timestamp + ) + + @classmethod + def diff_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of diff data into standard OrderBookMessage format + :param record: a row of diff data from the database + :return: ProbitOrderBookMessage + """ + return ProbitOrderBookMessage( + message_type=OrderBookMessageType.DIFF, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def trade_message_from_exchange(cls, + msg: Dict[str, Any], + timestamp: Optional[float] = None, + metadata: Optional[Dict] = None): + """ + Convert a trade data into standard OrderBookMessage format + :param record: a trade data from the database + :return: ProbitOrderBookMessage + """ + + if metadata: + msg.update(metadata) + + msg.update({ + "exchange_order_id": msg.get("d"), + "trade_type": msg.get("s"), + "price": msg.get("p"), + "amount": msg.get("q"), + }) + + return ProbitOrderBookMessage( + message_type=OrderBookMessageType.TRADE, + content=msg, + timestamp=timestamp + ) + + @classmethod + def trade_message_from_db(cls, record: RowProxy, metadata: Optional[Dict] = None): + """ + *used for backtesting + Convert a row of trade data into standard OrderBookMessage format + :param record: a row of trade data from the database + :return: ProbitOrderBookMessage + """ + return ProbitOrderBookMessage( + message_type=OrderBookMessageType.TRADE, + content=record.json, + timestamp=record.timestamp + ) + + @classmethod + def from_snapshot(cls, snapshot: OrderBookMessage): + raise NotImplementedError(constants.EXCHANGE_NAME + " order book needs to retain individual order data.") + + @classmethod + def restore_from_snapshot_and_diffs(cls, snapshot: OrderBookMessage, diffs: List[OrderBookMessage]): + raise NotImplementedError(constants.EXCHANGE_NAME + " order book needs to retain individual order data.") diff --git a/hummingbot/connector/exchange/probit/probit_order_book_message.py b/hummingbot/connector/exchange/probit/probit_order_book_message.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hummingbot/connector/exchange/probit/probit_order_book_tracker.py b/hummingbot/connector/exchange/probit/probit_order_book_tracker.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hummingbot/connector/exchange/probit/probit_order_book_tracker_entry.py b/hummingbot/connector/exchange/probit/probit_order_book_tracker_entry.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hummingbot/connector/exchange/probit/probit_utils.py b/hummingbot/connector/exchange/probit/probit_utils.py new file mode 100644 index 0000000000..f070728dc7 --- /dev/null +++ b/hummingbot/connector/exchange/probit/probit_utils.py @@ -0,0 +1,18 @@ +from typing import ( + Any, + Dict, +) + +CENTRALIZED = True + +EXAMPLE_PAIR = "ETH-USDT" + +DEFAULT_FEES = [0.2, 0.2] + + +def convert_snapshot_message_to_order_book_row(message: Dict[str, Any]): + pass + + +def convert_diff_message_to_order_book_row(message: Dict[str, Any]): + pass From d911935f362687684201ed60b97d7661440729b4 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Mon, 15 Feb 2021 16:47:18 +0800 Subject: [PATCH 05/73] (add) probit dummy files --- hummingbot/connector/exchange/probit/dummy.pxd | 2 ++ hummingbot/connector/exchange/probit/dummy.pyx | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 hummingbot/connector/exchange/probit/dummy.pxd create mode 100644 hummingbot/connector/exchange/probit/dummy.pyx diff --git a/hummingbot/connector/exchange/probit/dummy.pxd b/hummingbot/connector/exchange/probit/dummy.pxd new file mode 100644 index 0000000000..4b098d6f59 --- /dev/null +++ b/hummingbot/connector/exchange/probit/dummy.pxd @@ -0,0 +1,2 @@ +cdef class dummy(): + pass diff --git a/hummingbot/connector/exchange/probit/dummy.pyx b/hummingbot/connector/exchange/probit/dummy.pyx new file mode 100644 index 0000000000..4b098d6f59 --- /dev/null +++ b/hummingbot/connector/exchange/probit/dummy.pyx @@ -0,0 +1,2 @@ +cdef class dummy(): + pass From 637c8e63bd65574ca6369e2c23f168e97ac89ca7 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Mon, 15 Feb 2021 20:39:58 +0800 Subject: [PATCH 06/73] (add) add ProbitOrderBookTracker[WIP] --- .../probit_api_order_book_data_source.py | 157 +++++++++++------- .../exchange/probit/probit_constants.py | 27 ++- .../exchange/probit/probit_order_book.py | 8 +- .../probit/probit_order_book_message.py | 89 ++++++++++ .../probit/probit_order_book_tracker.py | 108 ++++++++++++ .../connector/exchange/probit/probit_utils.py | 67 +++++++- 6 files changed, 381 insertions(+), 75 deletions(-) diff --git a/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py b/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py index 0bdb8ed737..33e41b86cb 100644 --- a/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py @@ -1,20 +1,27 @@ #!/usr/bin/env python +import aiohttp import asyncio import logging -import time -import aiohttp import pandas as pd +import time +import ujson +import websockets + import hummingbot.connector.exchange.probit.probit_constants as constants -from typing import Optional, List, Dict, Any +from typing import ( + Any, + AsyncIterable, + Dict, + List, + Optional, +) from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.order_book_message import OrderBookMessage from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource from hummingbot.logger import HummingbotLogger -from . import probit_utils -from .probit_order_book import ProbitOrderBook -from .probit_websocket import ProbitWebsocket -from .probit_utils import ms_timestamp_to_s +from hummingbot.connector.exchange.probit import probit_utils +from hummingbot.connector.exchange.probit.probit_order_book import ProbitOrderBook class ProbitAPIOrderBookDataSource(OrderBookTrackerDataSource): @@ -39,7 +46,7 @@ def __init__(self, trading_pairs: List[str] = None): async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, float]: result = {} async with aiohttp.ClientSession() as client: - async with client.get(f"{constants.TICKER_PATH_URL}") as response: + async with client.get(f"{constants.TICKER_URL}") as response: if response.status == 200: resp_json = await response.json() if "data" in resp_json: @@ -50,7 +57,7 @@ async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, flo @staticmethod async def fetch_trading_pairs() -> List[str]: async with aiohttp.ClientSession() as client: - async with client.get(f"{constants.MARKETS_PATH_URL}") as response: + async with client.get(f"{constants.MARKETS_URL}") as response: if response.status == 200: resp_json: Dict[str, Any] = await response.json() return [market["market_id"] for market in resp_json["data"]] @@ -62,7 +69,7 @@ async def get_order_book_data(trading_pair: str) -> Dict[str, any]: Get whole orderbook """ async with aiohttp.ClientSession() as client: - async with client.get(url=f"{constants.ORDER_BOOK_PATH_URL}", + async with client.get(url=f"{constants.ORDER_BOOK_URL}", params={"market_id": trading_pair}) as response: if response.status != 200: raise IOError( @@ -84,41 +91,65 @@ async def get_new_order_book(self, trading_pair: str) -> OrderBook: order_book.apply_snapshot(bids, asks, snapshot_msg.update_id) return order_book + async def _inner_messages(self, + ws: websockets.WebSocketClientProtocol) -> AsyncIterable[str]: + try: + while True: + msg: str = await asyncio.wait_for(ws.recv(), timeout=self.MESSAGE_TIMEOUT) + yield msg + except asyncio.TimeoutError: + try: + pong_waiter = await ws.ping() + await asyncio.wait_for(pong_waiter, timeout=self.PING_TIMEOUT) + except asyncio.TimeoutError: + raise + except websockets.exceptions.ConnectionClosed: + return + finally: + await ws.close() + + async def listen_for_order_book_diffs_trades(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): + # TODO: Combine both trades and order_book_diffs + # params: Dict[str, Any] = { + # "channel": "marketdata", + # "filter": ["order_books","recent_trades"], + # "interval": 100, + # "market_id": trading_pair, + # "type": "subscribe" + # } + pass + async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): """ Listen for trades using websocket trade channel """ while True: try: - ws = ProbitWebsocket() - await ws.connect() - - await ws.subscribe(list(map( - lambda pair: f"trade.{probit_utils.convert_to_exchange_trading_pair(pair)}", - self._trading_pairs - ))) - - async for response in ws.on_message(): - if response.get("result") is None: - continue - - for trade in response["result"]["data"]: - trade: Dict[Any] = trade - trade_timestamp: int = ms_timestamp_to_s(trade["t"]) - trade_msg: OrderBookMessage = ProbitOrderBook.trade_message_from_exchange( - trade, - trade_timestamp, - metadata={"trading_pair": probit_utils.convert_from_exchange_trading_pair(trade["i"])} - ) - output.put_nowait(trade_msg) - + async with websockets.connect(uri=constants.WSS_URL) as ws: + ws: websockets.WebSocketClientProtocol = ws + for trading_pair in self._trading_pairs: + params: Dict[str, Any] = { + "channel": "marketdata", + "filter": ["recent_trades"], + "interval": 100, + "market_id": trading_pair, + "type": "subscribe" + } + await ws.send(ujson.dumps(params)) + async for raw_msg in self._inner_messages(ws): + msg = ujson.loads(raw_msg) + if "recent_trades" not in msg: + continue + for trade_entry in msg["recent_trades"]: + trade_msg: OrderBookMessage = ProbitOrderBook.trade_message_from_exchange(trade_entry) + output.put_nowait(trade_msg) except asyncio.CancelledError: raise except Exception: self.logger().error("Unexpected error.", exc_info=True) await asyncio.sleep(5.0) finally: - await ws.disconnect() + await ws.close() async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): """ @@ -126,31 +157,33 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp """ while True: try: - ws = ProbitWebsocket() - await ws.connect() - - await ws.subscribe(list(map( - lambda pair: f"book.{probit_utils.convert_to_exchange_trading_pair(pair)}.150", - self._trading_pairs - ))) - - async for response in ws.on_message(): - if response.get("result") is None: - continue - - order_book_data = response["result"]["data"][0] - timestamp: int = ms_timestamp_to_s(order_book_data["t"]) - # data in this channel is not order book diff but the entire order book (up to depth 150). - # so we need to convert it into a order book snapshot. - # Crypto.com does not offer order book diff ws updates. - orderbook_msg: OrderBookMessage = ProbitOrderBook.snapshot_message_from_exchange( - order_book_data, - timestamp, - metadata={"trading_pair": probit_utils.convert_from_exchange_trading_pair( - response["result"]["instrument_name"])} - ) - output.put_nowait(orderbook_msg) - + async with websockets.connect(uri=constants.WSS_URL) as ws: + ws: websockets.WebSocketClientProtocol = ws + for trading_pair in self._trading_pairs: + params: Dict[str, Any] = { + "channel": "marketdata", + "filter": ["order_books"], + "interval": 100, + "market_id": trading_pair, + "type": "subscribe" + } + await ws.send(ujson.dumps(params)) + async for raw_msg in self._inner_messages(ws): + msg_timestamp: int = int(time.time() * 1e3) + msg: Dict[str, Any] = ujson.loads(raw_msg) + if "order_books" not in msg: + continue + if "reset" in msg and msg["reset"] is True: + # First response from websocket is a snapshot. This is only when reset = True + snapshot_msg: OrderBookMessage = ProbitOrderBook.snapshot_message_from_exchange( + msg=msg, + timestamp=msg_timestamp, + ) + output.put_nowait(snapshot_msg) + for diff_entry in msg["order_books"]: + diff_msg: OrderBookMessage = ProbitOrderBook.diff_message_from_exchange(diff_entry, + msg_timestamp) + output.put_nowait(diff_msg) except asyncio.CancelledError: raise except Exception: @@ -162,7 +195,7 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp ) await asyncio.sleep(30.0) finally: - await ws.disconnect() + await ws.close() async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): """ @@ -173,10 +206,10 @@ async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, for trading_pair in self._trading_pairs: try: snapshot: Dict[str, any] = await self.get_order_book_data(trading_pair) - snapshot_timestamp: int = ms_timestamp_to_s(snapshot["t"]) + snapshot_timestamp: int = int(time.time() * 1e3) snapshot_msg: OrderBookMessage = ProbitOrderBook.snapshot_message_from_exchange( - snapshot, - snapshot_timestamp, + msg=snapshot, + timestamp=snapshot_timestamp, metadata={"trading_pair": trading_pair} ) output.put_nowait(snapshot_msg) diff --git a/hummingbot/connector/exchange/probit/probit_constants.py b/hummingbot/connector/exchange/probit/probit_constants.py index 5486551910..a471e48296 100644 --- a/hummingbot/connector/exchange/probit/probit_constants.py +++ b/hummingbot/connector/exchange/probit/probit_constants.py @@ -5,9 +5,26 @@ REST_URL = "https://api.probit.com/api/exchange/" WSS_URL = "wss://api.probit.com/api/exchange/v1/ws" -API_VERSON = "v1" +REST_API_VERSON = "v1" -TICKER_PATH_URL = f"{REST_URL+API_VERSON}/ticker" -MARKETS_PATH_URL = f"{REST_URL+API_VERSON}/market" -ORDER_BOOK_PATH_URL = f"{REST_URL+API_VERSON}/order_book" -NEW_ORDER_PATH_URL = f"{REST_URL+API_VERSON}/new_order" +# REST API Public Endpoints +TICKER_URL = f"{REST_URL+REST_API_VERSON}/ticker" +MARKETS_URL = f"{REST_URL+REST_API_VERSON}/market" +ORDER_BOOK_URL = f"{REST_URL+REST_API_VERSON}/order_book" +NEW_ORDER_URL = f"{REST_URL+REST_API_VERSON}/new_order" + +# REST API Private Endpoints +NEW_ORDER_URL = f"{REST_URL+REST_API_VERSON}/new_order" +CANCEL_ORDER_URL = f"{REST_URL+REST_API_VERSON}/cancel_order" +ORDER_HISTORY_URL = f"{REST_URL+REST_API_VERSON}/order_history" +TRADE_HISTORY_URL = f"{REST_URL+REST_API_VERSON}/trade_history" +BALANCE_URL = f"{REST_URL+REST_API_VERSON}/balance" +ORDER_URL = f"{REST_URL+REST_API_VERSON}/order" +OPEN_ORDER_URL = f"{REST_URL+REST_API_VERSON}/open_order" + +# Order Status Definitions +ORDER_STATUS = [ + "open", + "filled", + "cancelled", +] diff --git a/hummingbot/connector/exchange/probit/probit_order_book.py b/hummingbot/connector/exchange/probit/probit_order_book.py index 7217094899..7d9ae36e93 100644 --- a/hummingbot/connector/exchange/probit/probit_order_book.py +++ b/hummingbot/connector/exchange/probit/probit_order_book.py @@ -111,10 +111,10 @@ def trade_message_from_exchange(cls, msg.update(metadata) msg.update({ - "exchange_order_id": msg.get("d"), - "trade_type": msg.get("s"), - "price": msg.get("p"), - "amount": msg.get("q"), + "exchange_order_id": msg.get("id"), + "trade_type": msg.get("side"), + "price": msg.get("price"), + "amount": msg.get("quantity"), }) return ProbitOrderBookMessage( diff --git a/hummingbot/connector/exchange/probit/probit_order_book_message.py b/hummingbot/connector/exchange/probit/probit_order_book_message.py index e69de29bb2..d2ee511b65 100644 --- a/hummingbot/connector/exchange/probit/probit_order_book_message.py +++ b/hummingbot/connector/exchange/probit/probit_order_book_message.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +from typing import ( + Dict, + List, + Optional, +) + +from hummingbot.core.data_type.order_book_row import OrderBookRow +from hummingbot.core.data_type.order_book_message import ( + OrderBookMessage, + OrderBookMessageType, +) + + +class ProbitOrderBookMessage(OrderBookMessage): + def __new__( + cls, + message_type: OrderBookMessageType, + content: Dict[str, any], + timestamp: Optional[float] = None, + *args, + **kwargs, + ): + if timestamp is None: + if message_type is OrderBookMessageType.SNAPSHOT: + raise ValueError("timestamp must not be None when initializing snapshot messages.") + timestamp = content["timestamp"] + + return super(ProbitOrderBookMessage, cls).__new__( + cls, message_type, content, timestamp=timestamp, *args, **kwargs + ) + + @property + def update_id(self) -> int: + if self.type in [OrderBookMessageType.DIFF, OrderBookMessageType.SNAPSHOT]: + return int(self.timestamp) + else: + return -1 + + @property + def trade_id(self) -> int: + if self.type is OrderBookMessageType.TRADE: + return int(self.timestamp) + return -1 + + @property + def trading_pair(self) -> str: + if "market_id" in self.content: + return self.content["market_id"] + elif "trading_pair" in self.content: + # Response for REST API does not include market_id. Instead we manually insert the trading_pair in listen_for_order_book_snapshots + return self.content["trading_pair"] + + @property + def asks(self) -> List[OrderBookRow]: + entries = [] + if "order_books" in self.content: # WS API response + entries = self.content["order_books"] + elif "data" in self.content: # REST API response + entries = self.content["data"] + + return [ + OrderBookRow(float(entry["price"]), float(entry["quantity"]), self.update_id) for entry in entries if entry["side"] == "sell" + ] + + @property + def bids(self) -> List[OrderBookRow]: + entries = [] + if "order_books" in self.content: # WS API response + entries = self.content["order_books"] + elif "data" in self.content: # REST API response + entries = self.content["data"] + + return [ + OrderBookRow(float(entry["price"]), float(entry["quantity"]), self.update_id) for entry in entries if entry["side"] == "buy" + ] + + def __eq__(self, other) -> bool: + return self.type == other.type and self.timestamp == other.timestamp + + def __lt__(self, other) -> bool: + if self.timestamp != other.timestamp: + return self.timestamp < other.timestamp + else: + """ + If timestamp is the same, the ordering is snapshot < diff < trade + """ + return self.type.value < other.type.value diff --git a/hummingbot/connector/exchange/probit/probit_order_book_tracker.py b/hummingbot/connector/exchange/probit/probit_order_book_tracker.py index e69de29bb2..8c0c1d0fad 100644 --- a/hummingbot/connector/exchange/probit/probit_order_book_tracker.py +++ b/hummingbot/connector/exchange/probit/probit_order_book_tracker.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +import asyncio +import bisect +import logging +import hummingbot.connector.exchange.probit.probit_constants as constants +import time + +from collections import defaultdict, deque +from typing import Optional, Dict, List, Deque +from hummingbot.core.data_type.order_book_message import OrderBookMessageType +from hummingbot.core.data_type.order_book_tracker import OrderBookTracker +from hummingbot.connector.exchange.probit import probit_utils +from hummingbot.connector.exchange.probit.probit_order_book_message import ProbitOrderBookMessage +from hummingbot.connector.exchange.probit.probit_api_order_book_data_source import ProbitAPIOrderBookDataSource +from hummingbot.connector.exchange.probit.probit_order_book import ProbitOrderBook +from hummingbot.logger import HummingbotLogger + + +class ProbitOrderBookTracker(OrderBookTracker): + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, trading_pairs: Optional[List[str]] = None,): + super().__init__(ProbitAPIOrderBookDataSource(trading_pairs), trading_pairs) + + self._ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + self._order_book_snapshot_stream: asyncio.Queue = asyncio.Queue() + self._order_book_diff_stream: asyncio.Queue = asyncio.Queue() + self._order_book_trade_stream: asyncio.Queue = asyncio.Queue() + self._process_msg_deque_task: Optional[asyncio.Task] = None + self._past_diffs_windows: Dict[str, Deque] = {} + self._order_books: Dict[str, ProbitOrderBook] = {} + self._saved_message_queues: Dict[str, Deque[ProbitOrderBookMessage]] = \ + defaultdict(lambda: deque(maxlen=1000)) + self._order_book_stream_listener_task: Optional[asyncio.Task] = None + self._order_book_trade_listener_task: Optional[asyncio.Task] = None + + @property + def exchange_name(self) -> str: + """ + Name of the current exchange + """ + return constants.EXCHANGE_NAME + + async def _track_single_book(self, trading_pair: str): + """ + Update an order book with changes from the latest batch of received messages + """ + past_diffs_window: Deque[ProbitOrderBookMessage] = deque() + self._past_diffs_windows[trading_pair] = past_diffs_window + + message_queue: asyncio.Queue = self._tracking_message_queues[trading_pair] + order_book: ProbitOrderBook = self._order_books[trading_pair] + + last_message_timestamp: float = time.time() + diff_messages_accepted: int = 0 + + while True: + try: + message: ProbitOrderBookMessage = None + saved_messages: Deque[ProbitOrderBookMessage] = self._saved_message_queues[trading_pair] + # Process saved messages first if there are any + if len(saved_messages) > 0: + message = saved_messages.popleft() + else: + message = await message_queue.get() + + if message.type is OrderBookMessageType.DIFF: + bids, asks = probit_utils.convert_diff_message_to_order_book_row(message) + order_book.apply_diffs(bids, asks, message.update_id) + past_diffs_window.append(message) + while len(past_diffs_window) > self.PAST_DIFF_WINDOW_SIZE: + past_diffs_window.popleft() + diff_messages_accepted += 1 + + # Output some statistics periodically. + now: float = time.time() + if int(now / 60.0) > int(last_message_timestamp / 60.0): + self.logger().debug("Processed %d order book diffs for %s.", + diff_messages_accepted, trading_pair) + diff_messages_accepted = 0 + last_message_timestamp = now + elif message.type is OrderBookMessageType.SNAPSHOT: + past_diffs: List[ProbitOrderBookMessage] = list(past_diffs_window) + # only replay diffs later than snapshot, first update active order with snapshot then replay diffs + replay_position = bisect.bisect_right(past_diffs, message) + replay_diffs = past_diffs[replay_position:] + s_bids, s_asks = probit_utils.convert_snapshot_message_to_order_book_row(message) + order_book.apply_snapshot(s_bids, s_asks, message.update_id) + for diff_message in replay_diffs: + d_bids, d_asks = probit_utils.convert_diff_message_to_order_book_row(diff_message) + order_book.apply_diffs(d_bids, d_asks, diff_message.update_id) + + self.logger().debug("Processed order book snapshot for %s.", trading_pair) + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + f"Unexpected error processing order book messages for {trading_pair}.", + exc_info=True, + app_warning_msg="Unexpected error processing order book messages. Retrying after 5 seconds." + ) + await asyncio.sleep(5.0) diff --git a/hummingbot/connector/exchange/probit/probit_utils.py b/hummingbot/connector/exchange/probit/probit_utils.py index f070728dc7..f1ad3911a2 100644 --- a/hummingbot/connector/exchange/probit/probit_utils.py +++ b/hummingbot/connector/exchange/probit/probit_utils.py @@ -1,8 +1,19 @@ +#!/usr/bin/env python + from typing import ( Any, Dict, + List, + Tuple ) +from hummingbot.core.utils.tracking_nonce import get_tracking_nonce +from hummingbot.core.data_type.order_book_row import OrderBookRow +from hummingbot.core.data_type.order_book_message import OrderBookMessage + +from hummingbot.client.config.config_var import ConfigVar +from hummingbot.client.config.config_methods import using_exchange + CENTRALIZED = True EXAMPLE_PAIR = "ETH-USDT" @@ -10,9 +21,57 @@ DEFAULT_FEES = [0.2, 0.2] -def convert_snapshot_message_to_order_book_row(message: Dict[str, Any]): - pass +def get_new_client_order_id(is_buy: bool, trading_pair: str) -> str: + side = "B" if is_buy else "S" + return f"{side}-{trading_pair}-{get_tracking_nonce()}" + + +def convert_snapshot_message_to_order_book_row(message: OrderBookMessage) -> Tuple[List[OrderBookRow], List[OrderBookRow]]: + update_id = message.update_id + data = [] + if "data" in message.content: # From REST API + data: List[Dict[str, Any]] = message.content["data"] + elif "order_books" in message.content: # From Websocket API + data: List[Dict[str, Any]] = message.content["order_books"] + bids, asks = [], [] + + for entry in data: + order_row = OrderBookRow(entry["price"], entry["quantity"], update_id) + if entry["side"] == "buy": + bids.append(order_row) + else: # entry["type"] == "Sell": + asks.append(order_row) + + return bids, asks + + +def convert_diff_message_to_order_book_row(message: OrderBookMessage) -> Tuple[List[OrderBookRow], List[OrderBookRow]]: + update_id = message.update_id + data = message.content["order_books"] + bids = [] + asks = [] + + for entry in data: + order_row = OrderBookRow(entry["price"], entry["quantity"], update_id) + if entry["side"] == "buy": + bids.append(order_row) + elif entry["side"] == "sell": + asks.append(order_row) + + return bids, asks -def convert_diff_message_to_order_book_row(message: Dict[str, Any]): - pass +KEYS = { + "probit_api_key": + ConfigVar(key="probit_api_key", + prompt="Enter your ProBit API key >>> ", + required_if=using_exchange("probit"), + is_secure=True, + is_connect_key=True), + "probit_secret_key": + ConfigVar(key="probit_secret_key", + prompt="Enter your ProBit secret key >>> ", + required_if=using_exchange("probit"), + is_secure=True, + is_connect_key=True), +} From a55ded574cca7d687b52e6a6e8cf16839b08abf1 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Mon, 15 Feb 2021 21:22:30 +0800 Subject: [PATCH 07/73] (fix) include additional handling of ws messages and fix timestamp issue in listen_for_trades --- .../probit/probit_api_order_book_data_source.py | 17 ++++++++++++++--- .../probit/probit_order_book_message.py | 5 ++--- .../connector/exchange/probit/probit_utils.py | 2 +- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py b/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py index 33e41b86cb..23a1a536f0 100644 --- a/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py @@ -80,7 +80,7 @@ async def get_order_book_data(trading_pair: str) -> Dict[str, any]: async def get_new_order_book(self, trading_pair: str) -> OrderBook: snapshot: Dict[str, Any] = await self.get_order_book_data(trading_pair) - snapshot_timestamp: float = time.time() + snapshot_timestamp: int = int(time.time() * 1e3) snapshot_msg: OrderBookMessage = ProbitOrderBook.snapshot_message_from_exchange( snapshot, snapshot_timestamp, @@ -137,11 +137,20 @@ async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asynci } await ws.send(ujson.dumps(params)) async for raw_msg in self._inner_messages(ws): + msg_timestamp: int = int(time.time() * 1e3) msg = ujson.loads(raw_msg) if "recent_trades" not in msg: + # Unrecognized response from "recent_trades" channel + continue + + if "reset" in msg and msg["reset"] is True: + # Ignores first response from "recent_trades" channel. This response details of the last 100 trades. continue + for trade_entry in msg["recent_trades"]: - trade_msg: OrderBookMessage = ProbitOrderBook.trade_message_from_exchange(trade_entry) + trade_msg: OrderBookMessage = ProbitOrderBook.trade_message_from_exchange( + msg=trade_entry, + timestamp=msg_timestamp) output.put_nowait(trade_msg) except asyncio.CancelledError: raise @@ -172,6 +181,7 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp msg_timestamp: int = int(time.time() * 1e3) msg: Dict[str, Any] = ujson.loads(raw_msg) if "order_books" not in msg: + # Unrecognized response from "order_books" channel continue if "reset" in msg and msg["reset"] is True: # First response from websocket is a snapshot. This is only when reset = True @@ -180,6 +190,7 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp timestamp=msg_timestamp, ) output.put_nowait(snapshot_msg) + continue for diff_entry in msg["order_books"]: diff_msg: OrderBookMessage = ProbitOrderBook.diff_message_from_exchange(diff_entry, msg_timestamp) @@ -210,7 +221,7 @@ async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, snapshot_msg: OrderBookMessage = ProbitOrderBook.snapshot_message_from_exchange( msg=snapshot, timestamp=snapshot_timestamp, - metadata={"trading_pair": trading_pair} + metadata={"market_id": trading_pair} # Manually insert trading_pair here since API response does include trading pair ) output.put_nowait(snapshot_msg) self.logger().debug(f"Saved order book snapshot for {trading_pair}") diff --git a/hummingbot/connector/exchange/probit/probit_order_book_message.py b/hummingbot/connector/exchange/probit/probit_order_book_message.py index d2ee511b65..8325736e96 100644 --- a/hummingbot/connector/exchange/probit/probit_order_book_message.py +++ b/hummingbot/connector/exchange/probit/probit_order_book_message.py @@ -48,9 +48,8 @@ def trade_id(self) -> int: def trading_pair(self) -> str: if "market_id" in self.content: return self.content["market_id"] - elif "trading_pair" in self.content: - # Response for REST API does not include market_id. Instead we manually insert the trading_pair in listen_for_order_book_snapshots - return self.content["trading_pair"] + else: + raise ValueError("market_id not found in message content") @property def asks(self) -> List[OrderBookRow]: diff --git a/hummingbot/connector/exchange/probit/probit_utils.py b/hummingbot/connector/exchange/probit/probit_utils.py index f1ad3911a2..03a75b1b52 100644 --- a/hummingbot/connector/exchange/probit/probit_utils.py +++ b/hummingbot/connector/exchange/probit/probit_utils.py @@ -36,7 +36,7 @@ def convert_snapshot_message_to_order_book_row(message: OrderBookMessage) -> Tup bids, asks = [], [] for entry in data: - order_row = OrderBookRow(entry["price"], entry["quantity"], update_id) + order_row = OrderBookRow(float(entry["price"]), float(entry["quantity"]), update_id) if entry["side"] == "buy": bids.append(order_row) else: # entry["type"] == "Sell": From d60a9b3831b9af621787436756e2c1a8af63e9c6 Mon Sep 17 00:00:00 2001 From: vic-en Date: Mon, 15 Feb 2021 14:26:42 +0100 Subject: [PATCH 08/73] (feat) add _funding_payment_span to derivative connectors --- .../derivative/binance_perpetual/binance_perpetual_derivative.py | 1 + .../derivative/perpetual_finance/perpetual_finance_derivative.py | 1 + hummingbot/connector/derivative_base.py | 1 + 3 files changed, 3 insertions(+) diff --git a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py index b3b132693a..6e285a8d99 100644 --- a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py +++ b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py @@ -131,6 +131,7 @@ def __init__(self, self._trading_rules_polling_task = None self._last_poll_timestamp = 0 self._throttler = Throttler((10.0, 1.0)) + self._funding_payment_span = [0, 15] @property def name(self) -> str: diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py index 03626b6dd0..7e86a486d5 100644 --- a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py @@ -85,6 +85,7 @@ def __init__(self, self._auto_approve_task = None self._real_time_balance_update = False self._poll_notifier = None + self._funding_payment_span = [1800, 0] @property def name(self): diff --git a/hummingbot/connector/derivative_base.py b/hummingbot/connector/derivative_base.py index 59e4d7b64c..ff4750017a 100644 --- a/hummingbot/connector/derivative_base.py +++ b/hummingbot/connector/derivative_base.py @@ -19,6 +19,7 @@ def __init__(self): self._account_positions = {} self._position_mode = None self._leverage = 1 + self._funding_payment_span = [0, 0] # time span(in seconds) before and after funding period when exchanges consider active positions eligible for funding payment def set_position_mode(self, position_mode: PositionMode): """ From 541b86a4e46adcf512ce007f12fd0698c2826bc6 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Mon, 15 Feb 2021 21:30:47 +0800 Subject: [PATCH 09/73] (fix) include missing PING_TIMEOUT in ProbitAPIOrderBookDataSource --- .../exchange/probit/probit_api_order_book_data_source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py b/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py index 23a1a536f0..005600c462 100644 --- a/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py @@ -27,6 +27,7 @@ class ProbitAPIOrderBookDataSource(OrderBookTrackerDataSource): MAX_RETRIES = 20 MESSAGE_TIMEOUT = 30.0 + PING_TIMEOUT = 10.0 SNAPSHOT_TIMEOUT = 10.0 _logger: Optional[HummingbotLogger] = None @@ -144,7 +145,7 @@ async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asynci continue if "reset" in msg and msg["reset"] is True: - # Ignores first response from "recent_trades" channel. This response details of the last 100 trades. + # Ignores first response from "recent_trades" channel. This response details the last 100 trades. continue for trade_entry in msg["recent_trades"]: From c745a7bc617bbcff8e6ac40ba2ee158cd3cd6137 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Mon, 15 Feb 2021 21:35:28 +0800 Subject: [PATCH 10/73] (add) add connector status for Probit Connector --- hummingbot/connector/connector_status.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hummingbot/connector/connector_status.py b/hummingbot/connector/connector_status.py index ccf1d44cfc..31b83553aa 100644 --- a/hummingbot/connector/connector_status.py +++ b/hummingbot/connector/connector_status.py @@ -20,6 +20,7 @@ 'kucoin': 'green', 'liquid': 'green', 'loopring': 'yellow', + 'probit': 'yellow', 'okex': 'green', 'terra': 'green' } From a82d378997a20abc1085b1a106ea87e6008a8685 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Tue, 16 Feb 2021 02:59:14 +0800 Subject: [PATCH 11/73] (add) ProbitAuth --- .../connector/exchange/probit/probit_auth.py | 88 +++++++++++++++++++ .../exchange/probit/probit_constants.py | 2 +- 2 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 hummingbot/connector/exchange/probit/probit_auth.py diff --git a/hummingbot/connector/exchange/probit/probit_auth.py b/hummingbot/connector/exchange/probit/probit_auth.py new file mode 100644 index 0000000000..c7dbda7912 --- /dev/null +++ b/hummingbot/connector/exchange/probit/probit_auth.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python + +import aiohttp +import base64 +import time +import ujson + +import hummingbot.connector.exchange.probit.probit_constants as constants + +from typing import Dict, Any + + +class ProbitAuth(): + """ + Auth class required by ProBit API + Learn more at https://docs-en.probit.com/docs/authorization-1 + """ + def __init__(self, api_key: str, secret_key: str): + self.api_key: str = api_key + self.secret_key: str = secret_key + self._oauth_token: str = None + self._oauth_token_expiration_time: int = -1 + self._http_client: aiohttp.ClientSession = aiohttp.ClientSession() + + def _token_has_expired(self): + now: int = int(time.time()) + return now >= self._oauth_token_expiration_time + + def _update_expiration_time(self, expiration_time: int): + self._oauth_token_expiration_time = expiration_time + + async def _generate_oauth_token(self) -> str: + try: + now: int = int(time.time()) + headers: Dict[str, Any] = self.get_headers() + payload = f"{self.api_key}:{self.secret_key}".encode() + b64_payload = base64.b64encode(payload).decode() + headers.update({ + "Authorization": f"Basic {b64_payload}" + }) + body = ujson.dumps({ + "grant_type": "client_credentials" + }) + resp = await self._http_client.post(url=constants.TOKEN_URL, + headers=headers, + data=body) + if resp.status != 200: + raise ValueError(f"{__name__}: Error occurred retrieving new OAuth Token. Response: {resp}") + + token_resp = await resp.json() + + # POST /token endpoint returns both access_token and expires_in + # Updates _oauth_token_expiration_time + + self._update_expiration_time(now + token_resp["expires_in"]) + return token_resp["access_token"] + except Exception as e: + raise e + + async def _get_oauth_token(self) -> str: + if self._oauth_token is None or self._token_has_expired(): + self._oauth_token = await self._generate_oauth_token() + return self._oauth_token + + async def generate_auth_dict(self): + """ + Generates authentication signature and return it in a dictionary along with other inputs + :return: a dictionary of request info including the request signature + """ + + headers = self.get_headers() + + access_token = await self._get_oauth_token() + headers.update({ + "Authorization": f"Bearer {access_token}" + }) + + return headers + + def get_headers(self) -> Dict[str, Any]: + """ + Generates authentication headers required by ProBit + :return: a dictionary of auth headers + """ + + return { + "Content-Type": 'application/json', + } diff --git a/hummingbot/connector/exchange/probit/probit_constants.py b/hummingbot/connector/exchange/probit/probit_constants.py index a471e48296..13991e6a11 100644 --- a/hummingbot/connector/exchange/probit/probit_constants.py +++ b/hummingbot/connector/exchange/probit/probit_constants.py @@ -11,7 +11,7 @@ TICKER_URL = f"{REST_URL+REST_API_VERSON}/ticker" MARKETS_URL = f"{REST_URL+REST_API_VERSON}/market" ORDER_BOOK_URL = f"{REST_URL+REST_API_VERSON}/order_book" -NEW_ORDER_URL = f"{REST_URL+REST_API_VERSON}/new_order" +TOKEN_URL = "https://accounts.probit.com/token" # REST API Private Endpoints NEW_ORDER_URL = f"{REST_URL+REST_API_VERSON}/new_order" From 5daf360cc002a50429d9efb6cb42bd8d63e46619 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Tue, 16 Feb 2021 20:39:06 +0800 Subject: [PATCH 12/73] (add) add ProbitAPIUserStreamDataSource --- .../probit_api_user_stream_data_source.py | 156 ++++++++++++++++++ .../connector/exchange/probit/probit_auth.py | 10 +- .../exchange/probit/probit_constants.py | 8 + 3 files changed, 169 insertions(+), 5 deletions(-) create mode 100644 hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py diff --git a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py new file mode 100644 index 0000000000..78596d685d --- /dev/null +++ b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python + +import asyncio +import logging +import ujson +import websockets + +import hummingbot.connector.exchange.probit.probit_constants as CONSTANTS + +from typing import ( + Any, + AsyncIterable, + Dict, + List, + Optional, +) + +from hummingbot.connector.exchange.probit.probit_auth import ProbitAuth +from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource +from hummingbot.logger import HummingbotLogger + + +class ProbitAPIUserStreamDataSource(UserStreamTrackerDataSource): + MAX_RETRIES = 20 + MESSAGE_TIMEOUT = 30.0 + + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, probit_auth: ProbitAuth, trading_pairs: Optional[List[str]] = []): + self._websocket_client: websockets.WebSocketClientProtocol = None + self._probit_auth: ProbitAuth = probit_auth + self._trading_pairs = trading_pairs + + self._last_recv_time: float = 0 + super().__init__() + + @property + def last_recv_time(self) -> float: + return self._last_recv_time + + async def _init_websocket_connection(self) -> websockets.WebSocketClientProtocol: + """ + Initialize WebSocket client for UserStreamDataSource + """ + try: + if self._websocket_client is None: + self._websocket_client = await websockets.connect(CONSTANTS.WSS_URL) + return self._websocket_client + except Exception: + self.logger().network("Unexpected error occured with ProBit WebSocket Connection") + + async def _authenticate(self, ws: websockets.WebSocketClientProtocol): + """ + Authenticates user to websocket + """ + while True: + try: + access_token: str = self._probit_auth.get_oauth_token() + auth_payload: Dict[str, Any] = { + "type": "authorization", + "token": access_token + } + await ws.send(ujson.dumps(auth_payload)) + auth_resp = await ws.recv() + auth_resp: Dict[str, Any] = ujson.loads(auth_resp) + + if auth_resp["result"] != "ok": + raise + else: + return + except asyncio.CancelledError: + raise + except Exception: + self.logger().info(f"Error occurred when authenticating to user stream. Response: {auth_resp}", + exc_info=True) + raise + + async def _subscribe_to_channels(self, ws: websockets.WebSocketClientProtocol): + """ + Subscribes to Private User Channels + """ + try: + for channel in CONSTANTS.WS_PRIVATE_CHANNELS: + sub_payload = { + "type": "subscribe", + "channel": channel + } + await ws.send(ujson.dumps(sub_payload)) + sub_resp = await ws.recv() + sub_resp: Dict[str, Any] = ujson.loads(sub_resp) + + if "reset" in sub_resp and sub_resp["reset"] is True: + continue + else: + self.logger().error(f"Error occured subscribing to {channel}...") + raise + + except asyncio.CancelledError: + raise + except Exception: + self.logger().error(f"Error occured subscribing to {CONSTANTS.EXCHANGE_NAME} private channels. " + f"Payload: {sub_payload} " + f"Resp: {sub_resp}", + exc_info=True) + + async def _inner_messages(self, + ws: websockets.WebSocketClientProtocol) -> AsyncIterable[str]: + try: + while True: + msg: str = await asyncio.wait_for(ws.recv()) + yield msg + except asyncio.TimeoutError: + try: + pong_waiter = await ws.ping() + await asyncio.wait_for(pong_waiter, timeout=self.PING_TIMEOUT) + except asyncio.TimeoutError: + raise + except websockets.exceptions.ConnectionClosed: + return + finally: + await ws.close() + + async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue) -> AsyncIterable[Any]: + """ + *required + Subscribe to user stream via web socket, and keep the connection open for incoming messages + :param ev_loop: ev_loop to execute this function in + :param output: an async queue where the incoming messages are stored + """ + + while True: + try: + ws: websockets.WebSocketClientProtocol = await self._init_websocket_connection() + self.logger().info("Authenticating to User Stream...") + await self._authenticate(ws) + self.logger().info("Successfully authenticated to User Stream.") + await self._subscribe_to_channels(ws) + self.logger().info("Successfully subscribed to all Private channels.") + + async for msg in self._inner_messages(ws): + print(f"{msg}") + output.put_nowait(msg) + except asyncio.CancelledError: + raise + except Exception: + self.logger().error( + "Unexpected error with Probit WebSocket connection. Retrying after 30 seconds...", + exc_info=True + ) + await asyncio.sleep(30.0) diff --git a/hummingbot/connector/exchange/probit/probit_auth.py b/hummingbot/connector/exchange/probit/probit_auth.py index c7dbda7912..d3a004f7fa 100644 --- a/hummingbot/connector/exchange/probit/probit_auth.py +++ b/hummingbot/connector/exchange/probit/probit_auth.py @@ -44,11 +44,11 @@ async def _generate_oauth_token(self) -> str: resp = await self._http_client.post(url=constants.TOKEN_URL, headers=headers, data=body) - if resp.status != 200: - raise ValueError(f"{__name__}: Error occurred retrieving new OAuth Token. Response: {resp}") - token_resp = await resp.json() + if resp.status != 200: + raise ValueError(f"{__name__}: Error occurred retrieving new OAuth Token. Response: {token_resp}") + # POST /token endpoint returns both access_token and expires_in # Updates _oauth_token_expiration_time @@ -57,7 +57,7 @@ async def _generate_oauth_token(self) -> str: except Exception as e: raise e - async def _get_oauth_token(self) -> str: + async def get_oauth_token(self) -> str: if self._oauth_token is None or self._token_has_expired(): self._oauth_token = await self._generate_oauth_token() return self._oauth_token @@ -70,7 +70,7 @@ async def generate_auth_dict(self): headers = self.get_headers() - access_token = await self._get_oauth_token() + access_token = await self.get_oauth_token() headers.update({ "Authorization": f"Bearer {access_token}" }) diff --git a/hummingbot/connector/exchange/probit/probit_constants.py b/hummingbot/connector/exchange/probit/probit_constants.py index 13991e6a11..95607932a6 100644 --- a/hummingbot/connector/exchange/probit/probit_constants.py +++ b/hummingbot/connector/exchange/probit/probit_constants.py @@ -22,6 +22,14 @@ ORDER_URL = f"{REST_URL+REST_API_VERSON}/order" OPEN_ORDER_URL = f"{REST_URL+REST_API_VERSON}/open_order" +# Websocket Private Channels +WS_PRIVATE_CHANNELS = [ + "open_order", + "order_history", + "trade_history", + "balance" +] + # Order Status Definitions ORDER_STATUS = [ "open", From 038a8cc9114de29c74a24bc1c30a3bb9dda3ca5e Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Tue, 16 Feb 2021 20:39:52 +0800 Subject: [PATCH 13/73] (remove) remove redundant prints --- .../exchange/probit/probit_api_user_stream_data_source.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py index 78596d685d..dc2adcd400 100644 --- a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py @@ -144,7 +144,6 @@ async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: a self.logger().info("Successfully subscribed to all Private channels.") async for msg in self._inner_messages(ws): - print(f"{msg}") output.put_nowait(msg) except asyncio.CancelledError: raise From 298574c8947f9f0444463cb3b04c1a1f4caf139a Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Tue, 16 Feb 2021 21:09:16 +0800 Subject: [PATCH 14/73] (fix) resolve minor issues with maintaining websocket connection --- .../probit_api_user_stream_data_source.py | 52 +++++++++---------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py index dc2adcd400..91654c1178 100644 --- a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py @@ -23,6 +23,7 @@ class ProbitAPIUserStreamDataSource(UserStreamTrackerDataSource): MAX_RETRIES = 20 MESSAGE_TIMEOUT = 30.0 + PING_TIMEOUT = 10.0 _logger: Optional[HummingbotLogger] = None @@ -59,27 +60,26 @@ async def _authenticate(self, ws: websockets.WebSocketClientProtocol): """ Authenticates user to websocket """ - while True: - try: - access_token: str = self._probit_auth.get_oauth_token() - auth_payload: Dict[str, Any] = { - "type": "authorization", - "token": access_token - } - await ws.send(ujson.dumps(auth_payload)) - auth_resp = await ws.recv() - auth_resp: Dict[str, Any] = ujson.loads(auth_resp) - - if auth_resp["result"] != "ok": - raise - else: - return - except asyncio.CancelledError: - raise - except Exception: - self.logger().info(f"Error occurred when authenticating to user stream. Response: {auth_resp}", - exc_info=True) + try: + access_token: str = await self._probit_auth.get_oauth_token() + auth_payload: Dict[str, Any] = { + "type": "authorization", + "token": access_token + } + await ws.send(ujson.dumps(auth_payload)) + auth_resp = await ws.recv() + auth_resp: Dict[str, Any] = ujson.loads(auth_resp) + + if auth_resp["result"] != "ok": + self.logger().error(f"Response: {auth_resp}", + exc_info=True) raise + except asyncio.CancelledError: + raise + except Exception: + self.logger().info("Error occurred when authenticating to user stream. ", + exc_info=True) + raise async def _subscribe_to_channels(self, ws: websockets.WebSocketClientProtocol): """ @@ -113,14 +113,8 @@ async def _inner_messages(self, ws: websockets.WebSocketClientProtocol) -> AsyncIterable[str]: try: while True: - msg: str = await asyncio.wait_for(ws.recv()) + msg: str = await ws.recv() yield msg - except asyncio.TimeoutError: - try: - pong_waiter = await ws.ping() - await asyncio.wait_for(pong_waiter, timeout=self.PING_TIMEOUT) - except asyncio.TimeoutError: - raise except websockets.exceptions.ConnectionClosed: return finally: @@ -144,6 +138,7 @@ async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: a self.logger().info("Successfully subscribed to all Private channels.") async for msg in self._inner_messages(ws): + print(f"{msg}") output.put_nowait(msg) except asyncio.CancelledError: raise @@ -152,4 +147,7 @@ async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: a "Unexpected error with Probit WebSocket connection. Retrying after 30 seconds...", exc_info=True ) + if self._websocket_client is not None: + await self._websocket_client.close() + self._websocket_client = None await asyncio.sleep(30.0) From cc081fc68aeebfcea7db53f6d6a21527f69eddfa Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Tue, 16 Feb 2021 21:25:09 +0800 Subject: [PATCH 15/73] (add) add ProbitUserStreamTracker --- .../probit_api_user_stream_data_source.py | 1 - .../probit/probit_user_stream_tracker.py | 73 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 hummingbot/connector/exchange/probit/probit_user_stream_tracker.py diff --git a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py index 91654c1178..e4484308e1 100644 --- a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py @@ -138,7 +138,6 @@ async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: a self.logger().info("Successfully subscribed to all Private channels.") async for msg in self._inner_messages(ws): - print(f"{msg}") output.put_nowait(msg) except asyncio.CancelledError: raise diff --git a/hummingbot/connector/exchange/probit/probit_user_stream_tracker.py b/hummingbot/connector/exchange/probit/probit_user_stream_tracker.py new file mode 100644 index 0000000000..d1bcb1cbb8 --- /dev/null +++ b/hummingbot/connector/exchange/probit/probit_user_stream_tracker.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +import asyncio +import logging + +from typing import ( + Optional, + List, +) + +from hummingbot.connector.exchange.probit.probit_auth import ProbitAuth +from hummingbot.connector.exchange.probit.probit_constants import EXCHANGE_NAME +from hummingbot.connector.exchange.probit.probit_api_user_stream_data_source import \ + ProbitAPIUserStreamDataSource +from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource +from hummingbot.core.data_type.user_stream_tracker import UserStreamTracker +from hummingbot.core.utils.async_utils import ( + safe_ensure_future, + safe_gather, +) +from hummingbot.logger import HummingbotLogger + + +class ProbitUserStreamTracker(UserStreamTracker): + _logger: Optional[HummingbotLogger] = None + + @classmethod + def logger(cls) -> HummingbotLogger: + if cls._logger is None: + cls._logger = logging.getLogger(__name__) + return cls._logger + + def __init__(self, + probit_auth: Optional[ProbitAuth] = None, + trading_pairs: Optional[List[str]] = []): + super().__init__() + self._probit_auth: ProbitAuth = probit_auth + self._trading_pairs: List[str] = trading_pairs + self._ev_loop: asyncio.events.AbstractEventLoop = asyncio.get_event_loop() + self._data_source: Optional[UserStreamTrackerDataSource] = None + self._user_stream_tracking_task: Optional[asyncio.Task] = None + + @property + def data_source(self) -> UserStreamTrackerDataSource: + """ + *required + Initializes a user stream data source (user specific order diffs from live socket stream) + :return: OrderBookTrackerDataSource + """ + if not self._data_source: + self._data_source = ProbitAPIUserStreamDataSource( + probit_auth=self._probit_auth, + trading_pairs=self._trading_pairs + ) + return self._data_source + + @property + def exchange_name(self) -> str: + """ + *required + Name of the current exchange + """ + return EXCHANGE_NAME + + async def start(self): + """ + *required + Start all listeners and tasks + """ + self._user_stream_tracking_task = safe_ensure_future( + self.data_source.listen_for_user_stream(self._ev_loop, self._user_stream) + ) + await safe_gather(self._user_stream_tracking_task) From e4e2834990632365a0bba78a3399ef341e869bc1 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Wed, 17 Feb 2021 00:05:00 +0800 Subject: [PATCH 16/73] (add) add ProbitInFlightOrder --- .../exchange/probit/probit_in_flight_order.py | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 hummingbot/connector/exchange/probit/probit_in_flight_order.py diff --git a/hummingbot/connector/exchange/probit/probit_in_flight_order.py b/hummingbot/connector/exchange/probit/probit_in_flight_order.py new file mode 100644 index 0000000000..4381ee8549 --- /dev/null +++ b/hummingbot/connector/exchange/probit/probit_in_flight_order.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python + +import asyncio + +from decimal import Decimal +from typing import ( + Any, + Dict, + Optional, +) + +from hummingbot.connector.in_flight_order_base import InFlightOrderBase +from hummingbot.core.event.events import ( + OrderType, + TradeType +) + + +class ProbitInFlightOrder(InFlightOrderBase): + def __init__(self, + client_order_id: str, + exchange_order_id: Optional[str], + trading_pair: str, + order_type: OrderType, + trade_type: TradeType, + price: Decimal, + amount: Decimal, + initial_state: str = "open"): + super().__init__( + client_order_id, + exchange_order_id, + trading_pair, + order_type, + trade_type, + price, + amount, + initial_state, + ) + self.trade_id_set = set() + self.cancelled_event = asyncio.Event() + + @property + def is_done(self) -> bool: + return self.last_state in {"filled", "cancelled"} + + @property + def is_failure(self) -> bool: + # TODO: Determine Order Status Definitions for failed orders + return self.last_state in {"REJECTED"} + + @property + def is_cancelled(self) -> bool: + return self.last_state in {"cancelled"} + + @classmethod + def from_json(cls, data: Dict[str, Any]) -> InFlightOrderBase: + """ + :param data: json data from API + :return: formatted InFlightOrder + """ + retval = ProbitInFlightOrder( + data["client_order_id"], + data["exchange_order_id"], + data["trading_pair"], + getattr(OrderType, data["order_type"]), + getattr(TradeType, data["trade_type"]), + Decimal(data["price"]), + Decimal(data["amount"]), + data["last_state"] + ) + retval.executed_amount_base = Decimal(data["executed_amount_base"]) + retval.executed_amount_quote = Decimal(data["executed_amount_quote"]) + retval.fee_asset = data["fee_asset"] + retval.fee_paid = Decimal(data["fee_paid"]) + retval.last_state = data["last_state"] + return retval + + def update_with_trade_update(self, trade_update: Dict[str, Any]) -> bool: + """ + Updates the in flight order with trade update (from private/get-order-detail end point) + return: True if the order gets updated otherwise False + """ + trade_id = trade_update["id"] + # trade_update["orderId"] is type int + if str(trade_update["order_id"]) != self.exchange_order_id or trade_id in self.trade_id_set: + # trade already recorded + return False + self.trade_id_set.add(trade_id) + self.executed_amount_base += Decimal(str(trade_update["quantity"])) + self.fee_paid += Decimal(str(trade_update["fee_amount"])) + self.executed_amount_quote += (Decimal(str(trade_update["price"])) * + Decimal(str(trade_update["quantity"]))) + if not self.fee_asset: + self.fee_asset = trade_update["fee_currency_id"] + return True From 8bd84df3831368543079171b72d5c2d8f4d81781 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Wed, 17 Feb 2021 07:17:13 +0800 Subject: [PATCH 17/73] (add) ProbitExchange[WIP] --- .../exchange/probit/probit_exchange.py | 852 ++++++++++++++++++ 1 file changed, 852 insertions(+) create mode 100644 hummingbot/connector/exchange/probit/probit_exchange.py diff --git a/hummingbot/connector/exchange/probit/probit_exchange.py b/hummingbot/connector/exchange/probit/probit_exchange.py new file mode 100644 index 0000000000..eeeb3430f5 --- /dev/null +++ b/hummingbot/connector/exchange/probit/probit_exchange.py @@ -0,0 +1,852 @@ +#!/usr/bin/env python + +import aiohttp +import asyncio +import logging +import math +import time + +from decimal import Decimal +from typing import ( + Dict, + List, + Optional, + Any, + AsyncIterable, +) + +from hummingbot.connector.exchange_base import ExchangeBase +from hummingbot.connector.exchange.probit import probit_constants as CONSTANTS +from hummingbot.connector.exchange.probit import probit_utils +from hummingbot.connector.exchange.probit.probit_auth import ProbitAuth +from hummingbot.connector.exchange.probit.probit_in_flight_order import ProbitInFlightOrder +from hummingbot.connector.exchange.probit.probit_order_book_tracker import ProbitOrderBookTracker +from hummingbot.connector.exchange.probit.probit_user_stream_tracker import ProbitUserStreamTracker +from hummingbot.connector.trading_rule import TradingRule +from hummingbot.core.clock import Clock +from hummingbot.core.data_type.cancellation_result import CancellationResult +from hummingbot.core.data_type.common import OpenOrder +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.limit_order import LimitOrder +from hummingbot.core.event.events import ( + MarketEvent, + BuyOrderCompletedEvent, + SellOrderCompletedEvent, + OrderFilledEvent, + OrderCancelledEvent, + BuyOrderCreatedEvent, + SellOrderCreatedEvent, + MarketOrderFailureEvent, + OrderType, + TradeType, + TradeFee +) +from hummingbot.core.network_iterator import NetworkStatus +from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather +from hummingbot.logger import HummingbotLogger + +probit_logger = None +s_decimal_NaN = Decimal("nan") + + +class ProbitExchange(ExchangeBase): + """ + ProbitExchange connects with ProBit exchange and provides order book pricing, user account tracking and + trading functionality. + """ + API_CALL_TIMEOUT = 10.0 + SHORT_POLL_INTERVAL = 5.0 + UPDATE_ORDER_STATUS_MIN_INTERVAL = 10.0 + LONG_POLL_INTERVAL = 120.0 + + @classmethod + def logger(cls) -> HummingbotLogger: + global probit_logger + if probit_logger is None: + probit_logger = logging.getLogger(__name__) + return probit_logger + + def __init__(self, + probit_api_key: str, + probit_secret_key: str, + trading_pairs: Optional[List[str]] = None, + trading_required: bool = True + ): + """ + :param probit_api_key: The API key to connect to private ProBit APIs. + :param probit_secret_key: The API secret. + :param trading_pairs: The market trading pairs which to track order book data. + :param trading_required: Whether actual trading is needed. + """ + super().__init__() + self._trading_required = trading_required + self._trading_pairs = trading_pairs + self._probit_auth = ProbitAuth(probit_api_key, probit_secret_key) + self._order_book_tracker = ProbitOrderBookTracker(trading_pairs=trading_pairs) + self._user_stream_tracker = ProbitUserStreamTracker(self._probit_auth, trading_pairs) + self._ev_loop = asyncio.get_event_loop() + self._shared_client = None + self._poll_notifier = asyncio.Event() + self._last_timestamp = 0 + self._in_flight_orders = {} # Dict[client_order_id:str, ProbitInFlightOrder] + self._order_not_found_records = {} # Dict[client_order_id:str, count:int] + self._trading_rules = {} # Dict[trading_pair:str, TradingRule] + self._status_polling_task = None + self._user_stream_event_listener_task = None + self._trading_rules_polling_task = None + self._last_poll_timestamp = 0 + + @property + def name(self) -> str: + return CONSTANTS.EXCHANGE_NAME + + @property + def order_books(self) -> Dict[str, OrderBook]: + return self._order_book_tracker.order_books + + @property + def trading_rules(self) -> Dict[str, TradingRule]: + return self._trading_rules + + @property + def in_flight_orders(self) -> Dict[str, ProbitInFlightOrder]: + return self._in_flight_orders + + @property + def status_dict(self) -> Dict[str, bool]: + """ + A dictionary of statuses of various connector's components. + """ + return { + "order_books_initialized": self._order_book_tracker.ready, + "account_balance": len(self._account_balances) > 0 if self._trading_required else True, + "trading_rule_initialized": len(self._trading_rules) > 0, + "user_stream_initialized": + self._user_stream_tracker.data_source.last_recv_time > 0 if self._trading_required else True, + } + + @property + def ready(self) -> bool: + """ + :return True when all statuses pass, this might take 5-10 seconds for all the connector's components and + services to be ready. + """ + return all(self.status_dict.values()) + + @property + def limit_orders(self) -> List[LimitOrder]: + return [ + in_flight_order.to_limit_order() + for in_flight_order in self._in_flight_orders.values() + ] + + @property + def tracking_states(self) -> Dict[str, any]: + """ + :return active in-flight orders in json format, is used to save in sqlite db. + """ + return { + key: value.to_json() + for key, value in self._in_flight_orders.items() + if not value.is_done + } + + def restore_tracking_states(self, saved_states: Dict[str, any]): + """ + Restore in-flight orders from saved tracking states, this is st the connector can pick up on where it left off + when it disconnects. + :param saved_states: The saved tracking_states. + """ + self._in_flight_orders.update({ + key: ProbitInFlightOrder.from_json(value) + for key, value in saved_states.items() + }) + + def supported_order_types(self) -> List[OrderType]: + """ + :return a list of OrderType supported by this connector. + Note that Market order type is no longer required and will not be used. + """ + return [OrderType.LIMIT, OrderType.LIMIT_MAKER] + + def start(self, clock: Clock, timestamp: float): + """ + This function is called automatically by the clock. + """ + super().start(clock, timestamp) + + def stop(self, clock: Clock): + """ + This function is called automatically by the clock. + """ + super().stop(clock) + + async def start_network(self): + """ + This function is required by NetworkIterator base class and is called automatically. + It starts tracking order book, polling trading rules, + updating statuses and tracking user data. + """ + self._order_book_tracker.start() + self._trading_rules_polling_task = safe_ensure_future(self._trading_rules_polling_loop()) + if self._trading_required: + self._status_polling_task = safe_ensure_future(self._status_polling_loop()) + self._user_stream_tracker_task = safe_ensure_future(self._user_stream_tracker.start()) + self._user_stream_event_listener_task = safe_ensure_future(self._user_stream_event_listener()) + + async def stop_network(self): + """ + This function is required by NetworkIterator base class and is called automatically. + """ + self._order_book_tracker.stop() + if self._status_polling_task is not None: + self._status_polling_task.cancel() + self._status_polling_task = None + if self._trading_rules_polling_task is not None: + self._trading_rules_polling_task.cancel() + self._trading_rules_polling_task = None + if self._status_polling_task is not None: + self._status_polling_task.cancel() + self._status_polling_task = None + if self._user_stream_tracker_task is not None: + self._user_stream_tracker_task.cancel() + self._user_stream_tracker_task = None + if self._user_stream_event_listener_task is not None: + self._user_stream_event_listener_task.cancel() + self._user_stream_event_listener_task = None + + async def check_network(self) -> NetworkStatus: + """ + This function is required by NetworkIterator base class and is called periodically to check + the network connection. Simply ping the network (or call any light weight public API). + """ + try: + # since there is no ping endpoint, the lowest rate call is to get BTC-USDT ticker + await self._api_request("get", "public/get-ticker?instrument_name=BTC_USDT") + except asyncio.CancelledError: + raise + except Exception: + return NetworkStatus.NOT_CONNECTED + return NetworkStatus.CONNECTED + + async def _http_client(self) -> aiohttp.ClientSession: + """ + :returns Shared client session instance + """ + if self._shared_client is None: + self._shared_client = aiohttp.ClientSession() + return self._shared_client + + async def _trading_rules_polling_loop(self): + """ + Periodically update trading rule. + """ + while True: + try: + await self._update_trading_rules() + await asyncio.sleep(60) + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().network(f"Unexpected error while fetching trading rules. Error: {str(e)}", + exc_info=True, + app_warning_msg="Could not fetch new trading rules from ProBit. " + "Check network connection.") + await asyncio.sleep(0.5) + + async def _update_trading_rules(self): + market_info = await self._api_request("GET", path_url="public/get-instruments") + self._trading_rules.clear() + self._trading_rules = self._format_trading_rules(market_info) + + def _format_trading_rules(self, market_info: Dict[str, Any]) -> Dict[str, TradingRule]: + """ + Converts json API response into a dictionary of trading rules. + :param market_info: The json API response + :return A dictionary of trading rules. + Response Example: + { + data: [ + { + "id":"BCH-BTC", + "base_currency_id":"BCH", + "quote_currency_id":"BTC", + "min_price":"0.00000001", + "max_price":"9999999999999999", + "price_increment":"0.00000001", + "min_quantity":"0.00000001", + "max_quantity":"9999999999999999", + "quantity_precision":8, + "min_cost":"0", + "max_cost":"9999999999999999", + "cost_precision": 8 + }, + ... + ] + } + """ + result = {} + for market in market_info["data"]: + try: + trading_pair = market["id"] + + quantity_decimals = Decimal(str(market["quantity_precision"])) + quantity_step = Decimal("1") / Decimal(str(math.pow(10, quantity_decimals))) + + result[trading_pair] = TradingRule(trading_pair=trading_pair, + min_order_size=Decimal(str(market["min_cost"])), + max_order_size=Decimal(str(market["max_cost"])), + min_price_increment=Decimal(str(market["price_increment"])), + min_base_amount_increment=quantity_step) + except Exception: + self.logger().error(f"Error parsing the trading pair rule {market}. Skipping.", exc_info=True) + return result + + async def _api_request(self, + method: str, + path_url: str, + params: Optional[Dict[str, Any]] = None, + data: Optional[Dict[str, Any]] = None, + is_auth_required: bool = False) -> Dict[str, Any]: + """ + Sends an aiohttp request and waits for a response. + :param method: The HTTP method, e.g. get or post + :param path_url: The path url or the API end point + :param is_auth_required: Whether an authentication is required, when True the function will add encrypted + signature to the request. + :returns A response in json format. + """ + client = await self._http_client() + + if is_auth_required: + headers = self._probit_auth.generate_auth_dict() + else: + headers = self._probit_auth.get_headers() + + if method == "GET": + response = await client.get(path_url, headers=headers, params=params) + elif method == "POST": + response = await client.post(path_url, headers=headers, params=params, data=data) + else: + raise NotImplementedError(f"{method} HTTP Method not implemented. ") + + try: + parsed_response = await response.json() + except Exception as e: + raise IOError(f"Error parsing data from {path_url}. Error: {str(e)}") + if response.status != 200: + raise IOError(f"Error fetching data from {path_url}. HTTP status is {response.status}. " + f"Message: {parsed_response}") + if parsed_response["code"] != 0: + raise IOError(f"{path_url} API call failed, response: {parsed_response}") + return parsed_response + + def get_order_price_quantum(self, trading_pair: str, price: Decimal): + """ + Returns a price step, a minimum price increment for a given trading pair. + """ + trading_rule = self._trading_rules[trading_pair] + return trading_rule.min_price_increment + + def get_order_size_quantum(self, trading_pair: str, order_size: Decimal): + """ + Returns an order amount step, a minimum amount increment for a given trading pair. + """ + trading_rule = self._trading_rules[trading_pair] + return Decimal(trading_rule.min_base_amount_increment) + + def get_order_book(self, trading_pair: str) -> OrderBook: + if trading_pair not in self._order_book_tracker.order_books: + raise ValueError(f"No order book exists for '{trading_pair}'.") + return self._order_book_tracker.order_books[trading_pair] + + def buy(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, + price: Decimal = s_decimal_NaN, **kwargs) -> str: + """ + Buys an amount of base asset (of the given trading pair). This function returns immediately. + To see an actual order, you'll have to wait for BuyOrderCreatedEvent. + :param trading_pair: The market (e.g. BTC-USDT) to buy from + :param amount: The amount in base token value + :param order_type: The order type + :param price: The price (note: this is no longer optional) + :returns A new internal order id + """ + order_id: str = probit_utils.get_new_client_order_id(True, trading_pair) + safe_ensure_future(self._create_order(TradeType.BUY, order_id, trading_pair, amount, order_type, price)) + return order_id + + def sell(self, trading_pair: str, amount: Decimal, order_type=OrderType.MARKET, + price: Decimal = s_decimal_NaN, **kwargs) -> str: + """ + Sells an amount of base asset (of the given trading pair). This function returns immediately. + To see an actual order, you'll have to wait for SellOrderCreatedEvent. + :param trading_pair: The market (e.g. BTC-USDT) to sell from + :param amount: The amount in base token value + :param order_type: The order type + :param price: The price (note: this is no longer optional) + :returns A new internal order id + """ + order_id: str = probit_utils.get_new_client_order_id(False, trading_pair) + safe_ensure_future(self._create_order(TradeType.SELL, order_id, trading_pair, amount, order_type, price)) + return order_id + + def cancel(self, trading_pair: str, order_id: str): + """ + Cancel an order. This function returns immediately. + To get the cancellation result, you'll have to wait for OrderCancelledEvent. + :param trading_pair: The market (e.g. BTC-USDT) of the order. + :param order_id: The internal order id (also called client_order_id) + """ + safe_ensure_future(self._execute_cancel(trading_pair, order_id)) + return order_id + + async def _create_order(self, + trade_type: TradeType, + order_id: str, + trading_pair: str, + amount: Decimal, + order_type: OrderType, + price: Decimal): + """ + Calls create-order API end point to place an order, starts tracking the order and triggers order created event. + :param trade_type: BUY or SELL + :param order_id: Internal order id (also called client_order_id) + :param trading_pair: The market to place order + :param amount: The order amount (in base token value) + :param order_type: The order type + :param price: The order price + """ + if not order_type.is_limit_type(): + raise Exception(f"Unsupported order type: {order_type}") + trading_rule = self._trading_rules[trading_pair] + + amount = self.quantize_order_amount(trading_pair, amount) + price = self.quantize_order_price(trading_pair, price) + if amount < trading_rule.min_order_size: + raise ValueError(f"{trade_type.name} order amount {amount} is lower than the minimum order size " + f"{trading_rule.min_order_size}.") + + body_params = { + "market_id": trading_pair, + "type": "limit", # ProBit Order Types ["limit", "market"} + "side": trade_type.name.lower(), # ProBit Order Sides ["buy", "sell"] + "time_in_force": "gtc", # gtc = Good-Til-Cancelled + "limit_price": price, + "quantity": amount, + "client_order_id": order_id + } + + self.start_tracking_order(order_id, + None, + trading_pair, + trade_type, + price, + amount, + order_type + ) + try: + order_result = await self._api_request( + method="POST", + path_url=CONSTANTS.NEW_ORDER_URL, + data=body_params, + is_auth_required=True + ) + exchange_order_id = str(order_result["data"]["id"]) + tracked_order = self._in_flight_orders.get(order_id) + if tracked_order is not None: + self.logger().info(f"Created {order_type.name} {trade_type.name} order {order_id} for " + f"{amount} {trading_pair}.") + tracked_order.update_exchange_order_id(exchange_order_id) + + event_tag = MarketEvent.BuyOrderCreated if trade_type is TradeType.BUY else MarketEvent.SellOrderCreated + event_class = BuyOrderCreatedEvent if trade_type is TradeType.BUY else SellOrderCreatedEvent + self.trigger_event(event_tag, + event_class( + self.current_timestamp, + order_type, + trading_pair, + amount, + price, + order_id + )) + except asyncio.CancelledError: + raise + except Exception as e: + self.stop_tracking_order(order_id) + self.logger().network( + f"Error submitting {trade_type.name} {order_type.name} order to ProBit for " + f"{amount} {trading_pair} " + f"{price}.", + exc_info=True, + app_warning_msg=str(e) + ) + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent(self.current_timestamp, order_id, order_type)) + + def start_tracking_order(self, + order_id: str, + exchange_order_id: str, + trading_pair: str, + trade_type: TradeType, + price: Decimal, + amount: Decimal, + order_type: OrderType): + """ + Starts tracking an order by simply adding it into _in_flight_orders dictionary. + """ + self._in_flight_orders[order_id] = ProbitInFlightOrder( + client_order_id=order_id, + exchange_order_id=exchange_order_id, + trading_pair=trading_pair, + order_type=order_type, + trade_type=trade_type, + price=price, + amount=amount + ) + + def stop_tracking_order(self, order_id: str): + """ + Stops tracking an order by simply removing it from _in_flight_orders dictionary. + """ + if order_id in self._in_flight_orders: + del self._in_flight_orders[order_id] + + async def _execute_cancel(self, trading_pair: str, order_id: str) -> str: + """ + Executes order cancellation process by first calling cancel-order API. The API result doesn't confirm whether + the cancellation is successful, it simply states it receives the request. + :param trading_pair: The market trading pair + :param order_id: The internal order id + order.last_state to change to CANCELED + """ + try: + tracked_order = self._in_flight_orders.get(order_id) + if tracked_order is None: + raise ValueError(f"Failed to cancel order - {order_id}. Order not found.") + if tracked_order.exchange_order_id is None: + await tracked_order.get_exchange_order_id() + ex_order_id = tracked_order.exchange_order_id + + body_params = { + "market_id": trading_pair, + "order_id": ex_order_id + } + + await self._api_request( + method="POST", + path_url=CONSTANTS.CANCEL_ORDER_URL, + data=body_params, + is_auth_required=True + ) + return order_id + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().network( + f"Failed to cancel order {order_id}: {str(e)}", + exc_info=True, + app_warning_msg=f"Failed to cancel the order {order_id} on Probit. " + f"Check API key and network connection." + ) + + async def _status_polling_loop(self): + """ + Periodically update user balances and order status via REST API. This serves as a fallback measure for web + socket API updates. + """ + while True: + try: + self._poll_notifier = asyncio.Event() + await self._poll_notifier.wait() + await safe_gather( + self._update_balances(), + self._update_order_status(), + ) + self._last_poll_timestamp = self.current_timestamp + except asyncio.CancelledError: + raise + except Exception as e: + self.logger().error(str(e), exc_info=True) + self.logger().network("Unexpected error while fetching account updates.", + exc_info=True, + app_warning_msg="Could not fetch account updates from ProBit. " + "Check API key and network connection.") + await asyncio.sleep(0.5) + + async def _update_balances(self): + """ + Calls REST API to update total and available balances. + """ + local_asset_names = set(self._account_balances.keys()) + remote_asset_names = set() + balance_info = await self._api_request( + method="GET", + path_url=CONSTANTS.BALANCE_URL, + is_auth_required=True + ) + for currency in balance_info["data"]: + asset_name = currency["currency_id"] + self._account_available_balances[asset_name] = Decimal(str(currency["available"])) + self._account_balances[asset_name] = Decimal(str(currency["total"])) + remote_asset_names.add(asset_name) + + asset_names_to_remove = local_asset_names.difference(remote_asset_names) + for asset_name in asset_names_to_remove: + del self._account_available_balances[asset_name] + del self._account_balances[asset_name] + + async def _update_order_status(self): + """ + Calls REST API to get status update for each in-flight order. + """ + last_tick = int(self._last_poll_timestamp / self.UPDATE_ORDER_STATUS_MIN_INTERVAL) + current_tick = int(self.current_timestamp / self.UPDATE_ORDER_STATUS_MIN_INTERVAL) + + if current_tick > last_tick and len(self._in_flight_orders) > 0: + # TODO: Refactor _update_order_status + tracked_orders = list(self._in_flight_orders.values()) + + tasks = [] + for tracked_order in tracked_orders: + ex_order_id = await tracked_order.get_exchange_order_id() + + query_params = { + "market_id": tracked_order.trading_pair, + "order_id": ex_order_id + } + + tasks.append(self._api_request(method="POST", + path_url=CONSTANTS.ORDER_URL, + params=query_params, + is_auth_required=True) + ) + self.logger().debug(f"Polling for order status updates of {len(tasks)} orders.") + update_results = await safe_gather(*tasks, return_exceptions=True) + for update_result in update_results: + if isinstance(update_result, Exception): + raise update_result + if "data" not in update_result: + self.logger().info(f"_update_order_status data not in resp: {update_result}") + continue + + # TODO: Determine best way to determine that order has been partially/fully executed + for trade_msg in update_result["result"]["trade_list"]: + await self._process_trade_message(trade_msg) + self._process_order_message(update_result["result"]["order_info"]) + + def _process_order_message(self, order_msg: Dict[str, Any]): + """ + Updates in-flight order and triggers cancellation or failure event if needed. + :param order_msg: The order response from either REST or web socket API (they are of the same format) + """ + client_order_id = order_msg["client_oid"] + if client_order_id not in self._in_flight_orders: + return + tracked_order = self._in_flight_orders[client_order_id] + # Update order execution status + tracked_order.last_state = order_msg["status"] + if tracked_order.is_cancelled: + self.logger().info(f"Successfully cancelled order {client_order_id}.") + self.trigger_event(MarketEvent.OrderCancelled, + OrderCancelledEvent( + self.current_timestamp, + client_order_id)) + tracked_order.cancelled_event.set() + self.stop_tracking_order(client_order_id) + elif tracked_order.is_failure: + self.logger().info(f"The market order {client_order_id} has failed according to order status API. " + f"Reason: {probit_utils.get_api_reason(order_msg['reason'])}") + self.trigger_event(MarketEvent.OrderFailure, + MarketOrderFailureEvent( + self.current_timestamp, + client_order_id, + tracked_order.order_type + )) + self.stop_tracking_order(client_order_id) + + async def _process_trade_message(self, trade_msg: Dict[str, Any]): + """ + Updates in-flight order and trigger order filled event for trade message received. Triggers order completed + event if the total executed amount equals to the specified order amount. + """ + for order in self._in_flight_orders.values(): + await order.get_exchange_order_id() + track_order = [o for o in self._in_flight_orders.values() if trade_msg["order_id"] == o.exchange_order_id] + if not track_order: + return + tracked_order = track_order[0] + updated = tracked_order.update_with_trade_update(trade_msg) + if not updated: + return + self.trigger_event( + MarketEvent.OrderFilled, + OrderFilledEvent( + self.current_timestamp, + tracked_order.client_order_id, + tracked_order.trading_pair, + tracked_order.trade_type, + tracked_order.order_type, + Decimal(str(trade_msg["traded_price"])), + Decimal(str(trade_msg["traded_quantity"])), + TradeFee(0.0, [(trade_msg["fee_currency"], Decimal(str(trade_msg["fee"])))]), + exchange_trade_id=trade_msg["order_id"] + ) + ) + if math.isclose(tracked_order.executed_amount_base, tracked_order.amount) or \ + tracked_order.executed_amount_base >= tracked_order.amount: + tracked_order.last_state = "FILLED" + self.logger().info(f"The {tracked_order.trade_type.name} order " + f"{tracked_order.client_order_id} has completed " + f"according to order status API.") + event_tag = MarketEvent.BuyOrderCompleted if tracked_order.trade_type is TradeType.BUY \ + else MarketEvent.SellOrderCompleted + event_class = BuyOrderCompletedEvent if tracked_order.trade_type is TradeType.BUY \ + else SellOrderCompletedEvent + self.trigger_event(event_tag, + event_class(self.current_timestamp, + tracked_order.client_order_id, + tracked_order.base_asset, + tracked_order.quote_asset, + tracked_order.fee_asset, + tracked_order.executed_amount_base, + tracked_order.executed_amount_quote, + tracked_order.fee_paid, + tracked_order.order_type)) + self.stop_tracking_order(tracked_order.client_order_id) + + async def cancel_all(self, timeout_seconds: float): + """ + Cancels all in-flight orders and waits for cancellation results. + Used by bot's top level stop and exit commands (cancelling outstanding orders on exit) + :param timeout_seconds: The timeout at which the operation will be canceled. + :returns List of CancellationResult which indicates whether each order is successfully cancelled. + """ + if self._trading_pairs is None: + raise Exception("cancel_all can only be used when trading_pairs are specified.") + cancellation_results = [] + try: + for trading_pair in self._trading_pairs: + await self._api_request( + "post", + "private/cancel-all-orders", + {"instrument_name": probit_utils.convert_to_exchange_trading_pair(trading_pair)}, + True + ) + open_orders = await self.get_open_orders() + for cl_order_id, tracked_order in self._in_flight_orders.items(): + open_order = [o for o in open_orders if o.client_order_id == cl_order_id] + if not open_order: + cancellation_results.append(CancellationResult(cl_order_id, True)) + self.trigger_event(MarketEvent.OrderCancelled, + OrderCancelledEvent(self.current_timestamp, cl_order_id)) + else: + cancellation_results.append(CancellationResult(cl_order_id, False)) + except Exception: + self.logger().network( + "Failed to cancel all orders.", + exc_info=True, + app_warning_msg="Failed to cancel all orders on ProBit. Check API key and network connection." + ) + return cancellation_results + + def tick(self, timestamp: float): + """ + Is called automatically by the clock for each clock's tick (1 second by default). + It checks if status polling task is due for execution. + """ + now = time.time() + poll_interval = (self.SHORT_POLL_INTERVAL + if now - self._user_stream_tracker.last_recv_time > 60.0 + else self.LONG_POLL_INTERVAL) + last_tick = int(self._last_timestamp / poll_interval) + current_tick = int(timestamp / poll_interval) + if current_tick > last_tick: + if not self._poll_notifier.is_set(): + self._poll_notifier.set() + self._last_timestamp = timestamp + + def get_fee(self, + base_currency: str, + quote_currency: str, + order_type: OrderType, + order_side: TradeType, + amount: Decimal, + price: Decimal = s_decimal_NaN) -> TradeFee: + """ + To get trading fee, this function is simplified by using fee override configuration. Most parameters to this + function are ignore except order_type. Use OrderType.LIMIT_MAKER to specify you want trading fee for + maker order. + """ + is_maker = order_type is OrderType.LIMIT_MAKER + return TradeFee(percent=self.estimate_fee_pct(is_maker)) + + async def _iter_user_event_queue(self) -> AsyncIterable[Dict[str, any]]: + while True: + try: + yield await self._user_stream_tracker.user_stream.get() + except asyncio.CancelledError: + raise + except Exception: + self.logger().network( + "Unknown error. Retrying after 1 seconds.", + exc_info=True, + app_warning_msg="Could not fetch user events from Probit. Check API key and network connection." + ) + await asyncio.sleep(1.0) + + async def _user_stream_event_listener(self): + """ + Listens to message in _user_stream_tracker.user_stream queue. The messages are put in by + ProbitAPIUserStreamDataSource. + """ + async for event_message in self._iter_user_event_queue(): + try: + if "result" not in event_message or "channel" not in event_message["result"]: + continue + channel = event_message["result"]["channel"] + if "user.trade" in channel: + for trade_msg in event_message["result"]["data"]: + await self._process_trade_message(trade_msg) + elif "user.order" in channel: + for order_msg in event_message["result"]["data"]: + self._process_order_message(order_msg) + elif channel == "user.balance": + balances = event_message["result"]["data"] + for balance_entry in balances: + asset_name = balance_entry["currency"] + self._account_balances[asset_name] = Decimal(str(balance_entry["balance"])) + self._account_available_balances[asset_name] = Decimal(str(balance_entry["available"])) + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error in user stream listener loop.", exc_info=True) + await asyncio.sleep(5.0) + + async def get_open_orders(self) -> List[OpenOrder]: + result = await self._api_request( + "post", + "private/get-open-orders", + {}, + True + ) + ret_val = [] + for order in result["result"]["order_list"]: + if probit_utils.HBOT_BROKER_ID not in order["client_oid"]: + continue + if order["type"] != "LIMIT": + raise Exception(f"Unsupported order type {order['type']}") + ret_val.append( + OpenOrder( + client_order_id=order["client_oid"], + trading_pair=probit_utils.convert_from_exchange_trading_pair(order["instrument_name"]), + price=Decimal(str(order["price"])), + amount=Decimal(str(order["quantity"])), + executed_amount=Decimal(str(order["cumulative_quantity"])), + status=order["status"], + order_type=OrderType.LIMIT, + is_buy=True if order["side"].lower() == "buy" else False, + time=int(order["create_time"]), + exchange_order_id=order["order_id"] + ) + ) + return ret_val From 1e4c9ec4057e83a9d966a43939454ecae38d4858 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Wed, 17 Feb 2021 15:03:48 +0800 Subject: [PATCH 18/73] (fix) fix websocket authentication error when oauth access token contains forward slash --- .../exchange/probit/probit_api_user_stream_data_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py index e4484308e1..9678c8fde2 100644 --- a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py @@ -68,7 +68,7 @@ async def _authenticate(self, ws: websockets.WebSocketClientProtocol): } await ws.send(ujson.dumps(auth_payload)) auth_resp = await ws.recv() - auth_resp: Dict[str, Any] = ujson.loads(auth_resp) + auth_resp: Dict[str, Any] = ujson.loads(auth_resp, escape_forward_slashes=False) if auth_resp["result"] != "ok": self.logger().error(f"Response: {auth_resp}", From 897938ea754a34367d0af692e5eb06016ced37ed Mon Sep 17 00:00:00 2001 From: Nullably <37262506+Nullably@users.noreply.github.com> Date: Wed, 17 Feb 2021 18:29:30 +0800 Subject: [PATCH 19/73] (feat) update allocation calculation to account for existing base/quote assets --- .../liquidity_mining/liquidity_mining.py | 45 ++++++++++++------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/hummingbot/strategy/liquidity_mining/liquidity_mining.py b/hummingbot/strategy/liquidity_mining/liquidity_mining.py index ad47b2e422..ca7b0c6b48 100644 --- a/hummingbot/strategy/liquidity_mining/liquidity_mining.py +++ b/hummingbot/strategy/liquidity_mining/liquidity_mining.py @@ -135,13 +135,11 @@ async def active_orders_df(self) -> pd.DataFrame: def market_status_df(self) -> pd.DataFrame: data = [] columns = ["Exchange", "Market", "Mid Price", "Volatility", "Base Bal", "Quote Bal", " Base %"] - balances = self.adjusted_available_balances() for market, market_info in self._market_infos.items(): - base, quote = market.split("-") mid_price = market_info.get_mid_price() base_bal = self._sell_budgets[market] quote_bal = self._buy_budgets[market] - total_bal = (base_bal * mid_price) + balances[quote] + total_bal = (base_bal * mid_price) + quote_bal base_pct = (base_bal * mid_price) / total_bal if total_bal > 0 else s_decimal_zero data.append([ self._exchange.display_name, @@ -201,23 +199,36 @@ def create_base_proposals(self): proposals.append(Proposal(market, PriceSize(buy_price, buy_size), PriceSize(sell_price, sell_size))) return proposals + def total_port_value_in_token(self) -> Decimal: + all_bals = self.adjusted_available_balances() + port_value = all_bals.get(self._token, s_decimal_zero) + for market, market_info in self._market_infos.items(): + base, quote = market.split("-") + if self.is_token_a_quote_token(): + port_value += all_bals[base] * market_info.get_mid_price() + else: + port_value += all_bals[quote] / market_info.get_mid_price() + return port_value + def create_budget_allocation(self): - # Equally assign buy and sell budgets to all markets + # Create buy and sell budgets for every market self._sell_budgets = {m: s_decimal_zero for m in self._market_infos} self._buy_budgets = {m: s_decimal_zero for m in self._market_infos} - token_bal = self.adjusted_available_balances().get(self._token, s_decimal_zero) - if self._token == list(self._market_infos.keys())[0].split("-")[0]: - base_markets = [m for m in self._market_infos if m.split("-")[0] == self._token] - sell_size = token_bal / len(base_markets) - for market in base_markets: - self._sell_budgets[market] = sell_size - self._buy_budgets[market] = self._exchange.get_available_balance(market.split("-")[1]) - else: - quote_markets = [m for m in self._market_infos if m.split("-")[1] == self._token] - buy_size = token_bal / len(quote_markets) - for market in quote_markets: - self._buy_budgets[market] = buy_size - self._sell_budgets[market] = self._exchange.get_available_balance(market.split("-")[0]) + port_value = self.total_port_value_in_token() + market_portion = port_value / len(self._market_infos) + balances = self.adjusted_available_balances() + for market, market_info in self._market_infos.items(): + base, quote = market.split("-") + if self.is_token_a_quote_token(): + self._sell_budgets[market] = balances[base] + buy_budget = market_portion - (balances[base] * market_info.get_mid_price()) + if buy_budget > s_decimal_zero: + self._buy_budgets[market] = buy_budget + else: + self._buy_budgets[market] = balances[quote] + sell_budget = market_portion - (balances[quote] / market_info.get_mid_price()) + if sell_budget > s_decimal_zero: + self._sell_budgets[market] = sell_budget def base_order_size(self, trading_pair: str, price: Decimal = s_decimal_zero): base, quote = trading_pair.split("-") From 9801e416eb8b1c1da52114ccb825bef964604c14 Mon Sep 17 00:00:00 2001 From: vic-en Date: Wed, 17 Feb 2021 15:39:41 +0100 Subject: [PATCH 20/73] (feat) add FundingPaymentCompletedEvent --- hummingbot/connector/markets_recorder.py | 30 ++++++++- hummingbot/core/event/events.py | 8 +++ hummingbot/model/funding_payment.py | 85 ++++++++++++++++++++++++ 3 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 hummingbot/model/funding_payment.py diff --git a/hummingbot/connector/markets_recorder.py b/hummingbot/connector/markets_recorder.py index 1b35fe333b..63fb3a1c93 100644 --- a/hummingbot/connector/markets_recorder.py +++ b/hummingbot/connector/markets_recorder.py @@ -27,6 +27,7 @@ MarketOrderFailureEvent, OrderCancelledEvent, OrderExpiredEvent, + FundingPaymentCompletedEvent, MarketEvent, TradeFee ) @@ -38,6 +39,7 @@ from hummingbot.model.order_status import OrderStatus from hummingbot.model.sql_connection_manager import SQLConnectionManager from hummingbot.model.trade_fill import TradeFill +from hummingbot.model.funding_payment import FundingPayment class MarketsRecorder: @@ -75,6 +77,7 @@ def __init__(self, self._fail_order_forwarder: SourceInfoEventForwarder = SourceInfoEventForwarder(self._did_fail_order) self._complete_order_forwarder: SourceInfoEventForwarder = SourceInfoEventForwarder(self._did_complete_order) self._expire_order_forwarder: SourceInfoEventForwarder = SourceInfoEventForwarder(self._did_expire_order) + self._funding_payment_forwarder: SourceInfoEventForwarder = SourceInfoEventForwarder(self._did_complete_funding_payment) self._event_pairs: List[Tuple[MarketEvent, SourceInfoEventForwarder]] = [ (MarketEvent.BuyOrderCreated, self._create_order_forwarder), @@ -84,7 +87,8 @@ def __init__(self, (MarketEvent.OrderFailure, self._fail_order_forwarder), (MarketEvent.BuyOrderCompleted, self._complete_order_forwarder), (MarketEvent.SellOrderCompleted, self._complete_order_forwarder), - (MarketEvent.OrderExpired, self._expire_order_forwarder) + (MarketEvent.OrderExpired, self._expire_order_forwarder), + (MarketEvent.FundingPaymentCompleted, self._funding_payment_forwarder) ] @property @@ -263,6 +267,30 @@ def _did_fill_order(self, market.add_trade_fills_from_market_recorder({TradeFillOrderDetails(trade_fill_record.market, trade_fill_record.exchange_trade_id, trade_fill_record.symbol)}) self.append_to_csv(trade_fill_record) + def _did_complete_funding_payment(self, + event_tag: int, + market: ConnectorBase, + evt: FundingPaymentCompletedEvent): + if threading.current_thread() != threading.main_thread(): + self._ev_loop.call_soon_threadsafe(self._did_complete_funding_payment, event_tag, market, evt) + return + + session: Session = self.session + timestamp: float = evt.timestamp + + # Try to find the funding payment has been recorded already. + payment_record: Optional[FundingPayment] = session.query(FundingPayment).filter(FundingPayment.timestamp == timestamp).one_or_none() + if payment_record is None: + funding_payment_record: FundingPayment = FundingPayment(timestamp=timestamp, + config_file_path=self.config_file_path, + market=market.display_name, + rate=evt.funding_rate, + symbol=evt.trading_pair, + amount=float(evt.amount)) + session.add(funding_payment_record) + session.commit() + # self.append_to_csv(funding_payment_record) + @staticmethod def _is_primitive_type(obj: object) -> bool: return not hasattr(obj, '__dict__') diff --git a/hummingbot/core/event/events.py b/hummingbot/core/event/events.py index 77a41a9689..a085e2ba32 100644 --- a/hummingbot/core/event/events.py +++ b/hummingbot/core/event/events.py @@ -203,6 +203,14 @@ class OrderExpiredEvent(NamedTuple): order_id: str +@dataclass +class FundingPaymentCompletedEvent: + timestamp: float + trading_pair: str + amount: Decimal + funding_rate: Decimal + + class MarketWithdrawAssetEvent(NamedTuple): timestamp: float tracking_id: str diff --git a/hummingbot/model/funding_payment.py b/hummingbot/model/funding_payment.py new file mode 100644 index 0000000000..99faa771a6 --- /dev/null +++ b/hummingbot/model/funding_payment.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +import pandas as pd +from typing import ( + List, + Optional, +) +from sqlalchemy import ( + Column, + Text, + Index, + BigInteger, + Float, +) +from sqlalchemy.orm import ( + Session +) +from datetime import datetime + +from . import HummingbotBase + + +class FundingPayment(HummingbotBase): + __tablename__ = "FundingPayment" + __table_args__ = (Index("tf_config_timestamp_index", + "config_file_path", "timestamp"), + Index("tf_market_trading_pair_timestamp_index", + "market", "symbol", "timestamp") + ) + + timestamp = Column(BigInteger, primary_key=True, nullable=False) + config_file_path = Column(Text, nullable=False) + market = Column(Text, nullable=False) + rate = Column(Float, nullable=False) + symbol = Column(Text, nullable=False) + amount = Column(Float, nullable=False) + + def __repr__(self) -> str: + return f"FundingPayment(timestamp={self.timestamp}, config_file_path='{self.config_file_path}', " \ + f"market='{self.market}', rate='{self.rate}' symbol='{self.symbol}', amount={self.amount}" + + @staticmethod + def get_funding_payments(sql_session: Session, + timestamp: str = None, + market: str = None, + trading_pair: str = None, + ) -> Optional[List["FundingPayment"]]: + filters = [] + if timestamp is not None: + filters.append(FundingPayment.timestamp == timestamp) + if market is not None: + filters.append(FundingPayment.market == market) + if trading_pair is not None: + filters.append(FundingPayment.symbol == trading_pair) + + payments: Optional[List[FundingPayment]] = (sql_session + .query(FundingPayment) + .filter(*filters) + .order_by(FundingPayment.timestamp.asc()) + .all()) + return payments + + @classmethod + def to_pandas(cls, payments: List): + columns: List[str] = ["Index", + "Timestamp", + "Exchange", + "Market", + "Rate", + "Amount"] + data = [] + index = 0 + for payment in payments: + index += 1 + data.append([ + index, + datetime.fromtimestamp(int(payment.timestamp / 1e3)).strftime("%Y-%m-%d %H:%M:%S"), + payment.market, + payment.rate, + payment.symbol, + payment.amount + ]) + df = pd.DataFrame(data=data, columns=columns) + df.set_index('Index', inplace=True) + + return df From 255f0f76718506dc9f60ac9098bcabe463500dd9 Mon Sep 17 00:00:00 2001 From: Nullably <37262506+Nullably@users.noreply.github.com> Date: Wed, 17 Feb 2021 23:15:31 +0800 Subject: [PATCH 21/73] (feat) add inventory_skew_enabled param (on by default) --- hummingbot/strategy/liquidity_mining/liquidity_mining.py | 5 ++++- .../liquidity_mining/liquidity_mining_config_map.py | 9 ++++++++- hummingbot/strategy/liquidity_mining/start.py | 2 ++ .../conf_liquidity_mining_strategy_TEMPLATE.yml | 5 ++++- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/hummingbot/strategy/liquidity_mining/liquidity_mining.py b/hummingbot/strategy/liquidity_mining/liquidity_mining.py index ca7b0c6b48..8045de06b2 100644 --- a/hummingbot/strategy/liquidity_mining/liquidity_mining.py +++ b/hummingbot/strategy/liquidity_mining/liquidity_mining.py @@ -39,6 +39,7 @@ def __init__(self, token: str, order_amount: Decimal, spread: Decimal, + inventory_skew_enabled: bool, target_base_pct: Decimal, order_refresh_time: float, order_refresh_tolerance_pct: Decimal, @@ -56,6 +57,7 @@ def __init__(self, self._spread = spread self._order_refresh_time = order_refresh_time self._order_refresh_tolerance_pct = order_refresh_tolerance_pct + self._inventory_skew_enabled = inventory_skew_enabled self._target_base_pct = target_base_pct self._inventory_range_multiplier = inventory_range_multiplier self._volatility_interval = volatility_interval @@ -100,7 +102,8 @@ def tick(self, timestamp: float): self.update_volatility() proposals = self.create_base_proposals() self._token_balances = self.adjusted_available_balances() - self.apply_inventory_skew(proposals) + if self._inventory_skew_enabled: + self.apply_inventory_skew(proposals) self.apply_budget_constraint(proposals) self.cancel_active_orders(proposals) self.execute_orders_proposal(proposals) diff --git a/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py b/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py index 891733234f..ece5a141de 100644 --- a/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py +++ b/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py @@ -4,7 +4,8 @@ from hummingbot.client.config.config_validators import ( validate_exchange, validate_decimal, - validate_int + validate_int, + validate_bool ) from hummingbot.client.settings import ( required_exchanges, @@ -65,6 +66,12 @@ def order_size_prompt() -> str: type_str="decimal", validator=lambda v: validate_decimal(v, 0, 100, inclusive=False), prompt_on_new=True), + "inventory_skew_enabled": + ConfigVar(key="inventory_skew_enabled", + prompt="Would you like to enable inventory skew? (Yes/No) >>> ", + type_str="bool", + default=True, + validator=validate_bool), "target_base_pct": ConfigVar(key="target_base_pct", prompt="For each pair, what is your target base asset percentage? (Enter 20 to indicate 20%) >>> ", diff --git a/hummingbot/strategy/liquidity_mining/start.py b/hummingbot/strategy/liquidity_mining/start.py index d99bb75720..03c7445276 100644 --- a/hummingbot/strategy/liquidity_mining/start.py +++ b/hummingbot/strategy/liquidity_mining/start.py @@ -14,6 +14,7 @@ def start(self): markets = quote_markets if quote_markets else base_markets order_amount = c_map.get("order_amount").value spread = c_map.get("spread").value / Decimal("100") + inventory_skew_enabled = c_map.get("inventory_skew_enabled").value target_base_pct = c_map.get("target_base_pct").value / Decimal("100") order_refresh_time = c_map.get("order_refresh_time").value order_refresh_tolerance_pct = c_map.get("order_refresh_tolerance_pct").value / Decimal("100") @@ -34,6 +35,7 @@ def start(self): token=token, order_amount=order_amount, spread=spread, + inventory_skew_enabled=inventory_skew_enabled, target_base_pct=target_base_pct, order_refresh_time=order_refresh_time, order_refresh_tolerance_pct=order_refresh_tolerance_pct, diff --git a/hummingbot/templates/conf_liquidity_mining_strategy_TEMPLATE.yml b/hummingbot/templates/conf_liquidity_mining_strategy_TEMPLATE.yml index cc841e7d4b..7daab8c8d4 100644 --- a/hummingbot/templates/conf_liquidity_mining_strategy_TEMPLATE.yml +++ b/hummingbot/templates/conf_liquidity_mining_strategy_TEMPLATE.yml @@ -2,7 +2,7 @@ ### Liquidity Mining strategy config ### ######################################################## -template_version: 1 +template_version: 2 strategy: null # The exchange to run this strategy. @@ -21,6 +21,9 @@ order_amount: null # The spread from mid price to place bid and ask orders, enter 1 to indicate 1% spread: null +# Whether to enable Inventory skew feature (true/false). +inventory_skew_enabled: null + # The target base asset percentage for all markets, enter 50 to indicate 50% target target_base_pct: null From e6dc3822030de806118a1705c5843b283f6365b4 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Thu, 18 Feb 2021 01:20:11 +0800 Subject: [PATCH 22/73] (add) ProbitExchange[pending initial dev testing] --- .../exchange/probit/probit_exchange.py | 204 +++++++++++------- .../exchange/probit/probit_in_flight_order.py | 4 +- .../connector/exchange/probit/probit_utils.py | 11 + 3 files changed, 144 insertions(+), 75 deletions(-) diff --git a/hummingbot/connector/exchange/probit/probit_exchange.py b/hummingbot/connector/exchange/probit/probit_exchange.py index eeeb3430f5..bf7fc2c977 100644 --- a/hummingbot/connector/exchange/probit/probit_exchange.py +++ b/hummingbot/connector/exchange/probit/probit_exchange.py @@ -603,7 +603,6 @@ async def _update_order_status(self): current_tick = int(self.current_timestamp / self.UPDATE_ORDER_STATUS_MIN_INTERVAL) if current_tick > last_tick and len(self._in_flight_orders) > 0: - # TODO: Refactor _update_order_status tracked_orders = list(self._in_flight_orders.values()) tasks = [] @@ -621,28 +620,66 @@ async def _update_order_status(self): is_auth_required=True) ) self.logger().debug(f"Polling for order status updates of {len(tasks)} orders.") - update_results = await safe_gather(*tasks, return_exceptions=True) - for update_result in update_results: + order_results: List[Dict[str, Any]] = await safe_gather(*tasks, return_exceptions=True) + + # Retrieve start_time and end_time of the earliest and last order. + # Retrieves all trades between this order creations. + min_order_ts: str = "" + + min_ts: float = float("inf") + for order_update in order_results: + order_ts: float = probit_utils.convert_iso_to_epoch(order_update["data"]["time"]) + + if order_ts < min_ts: + min_order_ts = order_update["data"]["time"] + min_ts = order_ts + + trade_history_tasks = [] + for trading_pair in self._trading_pairs: + query_params = { + "start_time": min_order_ts, + "end_time": probit_utils.get_iso_time_now(), + "limit": 1000, + "market_id": trading_pair + } + trade_history_tasks.append(self._api_request( + method="GET", + path_url=CONSTANTS.TRADE_HISTORY_URL + )) + trade_history_results: List[Dict[str, Any]] = await safe_gather(*trade_history_tasks, return_exceptions=True) + + for t_pair_history in trade_history_results: + if isinstance(t_pair_history, Exception): + raise t_pair_history + if "data" not in t_pair_history: + self.logger().info(f"Unexpected response from GET /trade_history. 'data' field not in resp: {t_pair_history}") + continue + + trade_details: List[Dict[str, Any]] = t_pair_history["data"] + for trade in trade_details: + self._process_trade_message(trade) + + for update_result in order_results: if isinstance(update_result, Exception): raise update_result if "data" not in update_result: self.logger().info(f"_update_order_status data not in resp: {update_result}") continue - # TODO: Determine best way to determine that order has been partially/fully executed - for trade_msg in update_result["result"]["trade_list"]: - await self._process_trade_message(trade_msg) - self._process_order_message(update_result["result"]["order_info"]) + order_details: List[Dict[str, Any]] = update_result["data"] + for order in order_details: + self._process_order_message(order_details) def _process_order_message(self, order_msg: Dict[str, Any]): """ - Updates in-flight order and triggers cancellation or failure event if needed. + Updates in-flight order and triggers trade, cancellation or failure event if needed. :param order_msg: The order response from either REST or web socket API (they are of the same format) """ - client_order_id = order_msg["client_oid"] + client_order_id = order_msg["client_order_id"] if client_order_id not in self._in_flight_orders: return tracked_order = self._in_flight_orders[client_order_id] + # Update order execution status tracked_order.last_state = order_msg["status"] if tracked_order.is_cancelled: @@ -655,7 +692,7 @@ def _process_order_message(self, order_msg: Dict[str, Any]): self.stop_tracking_order(client_order_id) elif tracked_order.is_failure: self.logger().info(f"The market order {client_order_id} has failed according to order status API. " - f"Reason: {probit_utils.get_api_reason(order_msg['reason'])}") + f"Order Message: {order_msg}") self.trigger_event(MarketEvent.OrderFailure, MarketOrderFailureEvent( self.current_timestamp, @@ -664,20 +701,27 @@ def _process_order_message(self, order_msg: Dict[str, Any]): )) self.stop_tracking_order(client_order_id) - async def _process_trade_message(self, trade_msg: Dict[str, Any]): + def _process_trade_message(self, order_msg: Dict[str, Any]): """ Updates in-flight order and trigger order filled event for trade message received. Triggers order completed event if the total executed amount equals to the specified order amount. """ - for order in self._in_flight_orders.values(): - await order.get_exchange_order_id() - track_order = [o for o in self._in_flight_orders.values() if trade_msg["order_id"] == o.exchange_order_id] - if not track_order: + ex_order_id = order_msg["order_id"] + + client_order_id = None + for track_order in self.in_flight_orders.values(): + if track_order.exchange_order_id == ex_order_id: + client_order_id = track_order.client_order_id + break + + if client_order_id is None: return - tracked_order = track_order[0] - updated = tracked_order.update_with_trade_update(trade_msg) + + tracked_order = self.in_flight_orders[client_order_id] + updated = tracked_order.update_with_trade_update(order_msg) if not updated: return + self.trigger_event( MarketEvent.OrderFilled, OrderFilledEvent( @@ -686,15 +730,15 @@ async def _process_trade_message(self, trade_msg: Dict[str, Any]): tracked_order.trading_pair, tracked_order.trade_type, tracked_order.order_type, - Decimal(str(trade_msg["traded_price"])), - Decimal(str(trade_msg["traded_quantity"])), - TradeFee(0.0, [(trade_msg["fee_currency"], Decimal(str(trade_msg["fee"])))]), - exchange_trade_id=trade_msg["order_id"] + Decimal(str(order_msg["price"])), + Decimal(str(order_msg["quantity"])), + TradeFee(0.0, [(order_msg["fee_currency_id"], Decimal(str(order_msg["fee_amount"])))]), + exchange_trade_id=order_msg["id"] ) ) if math.isclose(tracked_order.executed_amount_base, tracked_order.amount) or \ tracked_order.executed_amount_base >= tracked_order.amount: - tracked_order.last_state = "FILLED" + tracked_order.last_state = "filled" self.logger().info(f"The {tracked_order.trade_type.name} order " f"{tracked_order.client_order_id} has completed " f"according to order status API.") @@ -714,6 +758,41 @@ async def _process_trade_message(self, trade_msg: Dict[str, Any]): tracked_order.order_type)) self.stop_tracking_order(tracked_order.client_order_id) + async def get_open_orders(self) -> List[OpenOrder]: + ret_val = [] + for trading_pair in self._trading_pairs: + query_params = { + "market_id": trading_pair + } + result = await self._api_request( + method="GET", + path_url=CONSTANTS.OPEN_ORDER_URL, + params=query_params, + is_auth_required=True + ) + if "data" not in result: + self.logger().info(f"Unexpected response from GET {CONSTANTS.OPEN_ORDER_URL}. " + f"Params: {query_params} " + f"Response: {result} ") + for order in result["data"]["order_list"]: + if order["type"] != "limit": + raise Exception(f"Unsupported order type {order['type']}") + ret_val.append( + OpenOrder( + client_order_id=order["client_order_id"], + trading_pair=order["market_id"], + price=Decimal(str(order["limit_price"])), + amount=Decimal(str(order["quantity"])), + executed_amount=Decimal(str(order["quantity"])) - Decimal(str(order["filled_quantity"])), + status=order["status"], + order_type=OrderType.LIMIT, + is_buy=True if order["side"].lower() == "buy" else False, + time=int(probit_utils.convert_iso_to_epoch(order["time"])), + exchange_order_id=order["id"] + ) + ) + return ret_val + async def cancel_all(self, timeout_seconds: float): """ Cancels all in-flight orders and waits for cancellation results. @@ -725,13 +804,23 @@ async def cancel_all(self, timeout_seconds: float): raise Exception("cancel_all can only be used when trading_pairs are specified.") cancellation_results = [] try: - for trading_pair in self._trading_pairs: - await self._api_request( - "post", - "private/cancel-all-orders", - {"instrument_name": probit_utils.convert_to_exchange_trading_pair(trading_pair)}, - True - ) + + # ProBit does not have cancel_all_order endpoint + tasks = [] + for tracked_order in self.in_flight_orders.values(): + body_params = { + "market_id": tracked_order.trading_pair, + "order_id": tracked_order.exchange_order_id + } + tasks.append(self._api_request( + method="POST", + path_url=CONSTANTS.CANCEL_ORDER_URL, + data=body_params, + is_auth_required=True + )) + + await safe_gather(*tasks) + open_orders = await self.get_open_orders() for cl_order_id, tracked_order in self._in_flight_orders.items(): open_order = [o for o in open_orders if o.client_order_id == cl_order_id] @@ -801,52 +890,23 @@ async def _user_stream_event_listener(self): """ async for event_message in self._iter_user_event_queue(): try: - if "result" not in event_message or "channel" not in event_message["result"]: + if "channel" not in event_message or event_message["channel"] not in ["open_order", "order_history", "balance", "trade_history"]: continue - channel = event_message["result"]["channel"] - if "user.trade" in channel: - for trade_msg in event_message["result"]["data"]: - await self._process_trade_message(trade_msg) - elif "user.order" in channel: - for order_msg in event_message["result"]["data"]: - self._process_order_message(order_msg) - elif channel == "user.balance": - balances = event_message["result"]["data"] - for balance_entry in balances: - asset_name = balance_entry["currency"] - self._account_balances[asset_name] = Decimal(str(balance_entry["balance"])) - self._account_available_balances[asset_name] = Decimal(str(balance_entry["available"])) + channel = event_message["channel"] + + if channel == "balance": + for asset, balance_details in event_message["data"].items(): + self._account_balances[asset] = Decimal(str(balance_details["total"])) + self._account_available_balances[asset] = Decimal(str(balance_details["available"])) + elif channel in ["open_order", "order_history"]: + for order_update in event_message["data"]: + self._process_order_message(order_update) + elif channel == "trade_history": + for trade_update in event_message["data"]: + self._process_trade_message(trade_update) + except asyncio.CancelledError: raise except Exception: self.logger().error("Unexpected error in user stream listener loop.", exc_info=True) await asyncio.sleep(5.0) - - async def get_open_orders(self) -> List[OpenOrder]: - result = await self._api_request( - "post", - "private/get-open-orders", - {}, - True - ) - ret_val = [] - for order in result["result"]["order_list"]: - if probit_utils.HBOT_BROKER_ID not in order["client_oid"]: - continue - if order["type"] != "LIMIT": - raise Exception(f"Unsupported order type {order['type']}") - ret_val.append( - OpenOrder( - client_order_id=order["client_oid"], - trading_pair=probit_utils.convert_from_exchange_trading_pair(order["instrument_name"]), - price=Decimal(str(order["price"])), - amount=Decimal(str(order["quantity"])), - executed_amount=Decimal(str(order["cumulative_quantity"])), - status=order["status"], - order_type=OrderType.LIMIT, - is_buy=True if order["side"].lower() == "buy" else False, - time=int(order["create_time"]), - exchange_order_id=order["order_id"] - ) - ) - return ret_val diff --git a/hummingbot/connector/exchange/probit/probit_in_flight_order.py b/hummingbot/connector/exchange/probit/probit_in_flight_order.py index 4381ee8549..fbfddc36fc 100644 --- a/hummingbot/connector/exchange/probit/probit_in_flight_order.py +++ b/hummingbot/connector/exchange/probit/probit_in_flight_order.py @@ -77,13 +77,11 @@ def from_json(cls, data: Dict[str, Any]) -> InFlightOrderBase: def update_with_trade_update(self, trade_update: Dict[str, Any]) -> bool: """ - Updates the in flight order with trade update (from private/get-order-detail end point) + Updates the in flight order with trade update (from GET /trade_history end point) return: True if the order gets updated otherwise False """ trade_id = trade_update["id"] - # trade_update["orderId"] is type int if str(trade_update["order_id"]) != self.exchange_order_id or trade_id in self.trade_id_set: - # trade already recorded return False self.trade_id_set.add(trade_id) self.executed_amount_base += Decimal(str(trade_update["quantity"])) diff --git a/hummingbot/connector/exchange/probit/probit_utils.py b/hummingbot/connector/exchange/probit/probit_utils.py index 03a75b1b52..462a162f93 100644 --- a/hummingbot/connector/exchange/probit/probit_utils.py +++ b/hummingbot/connector/exchange/probit/probit_utils.py @@ -1,5 +1,8 @@ #!/usr/bin/env python +import dateutil.parser as dp + +from datetime import datetime from typing import ( Any, Dict, @@ -26,6 +29,14 @@ def get_new_client_order_id(is_buy: bool, trading_pair: str) -> str: return f"{side}-{trading_pair}-{get_tracking_nonce()}" +def convert_iso_to_epoch(ts: str) -> float: + return dp.parse(ts).timestamp() + + +def get_iso_time_now() -> str: + return datetime.utcnow().isoformat()[:-3] + 'Z' + + def convert_snapshot_message_to_order_book_row(message: OrderBookMessage) -> Tuple[List[OrderBookRow], List[OrderBookRow]]: update_id = message.update_id data = [] From 012bb8ca52ac079479377b7acf4c088cbf27bdda Mon Sep 17 00:00:00 2001 From: vic-en Date: Wed, 17 Feb 2021 18:49:30 +0100 Subject: [PATCH 23/73] (feat) add funding event listener in derivative connectors --- .../binance_perpetual_derivative.py | 19 +++--- .../perpetual_finance_derivative.py | 60 +++++++++++++++---- hummingbot/core/event/events.py | 1 + hummingbot/strategy/strategy_base.pxd | 1 + hummingbot/strategy/strategy_base.pyx | 12 ++++ hummingbot/strategy/strategy_py_base.pyx | 6 ++ 6 files changed, 77 insertions(+), 22 deletions(-) diff --git a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py index 6e285a8d99..ecb6d777a3 100644 --- a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py +++ b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py @@ -248,7 +248,7 @@ async def create_order(self, else: api_params["positionSide"] = "SHORT" if trade_type is TradeType.BUY else "LONG" - self.start_tracking_order(order_id, "", trading_pair, trade_type, price, amount, order_type, self._leverage, position_action.name) + self.start_tracking_order(order_id, "", trading_pair, trade_type, price, amount, order_type, self._leverage[trading_pair], position_action.name) try: order_result = await self.request(path="/fapi/v1/order", @@ -273,7 +273,7 @@ async def create_order(self, amount, price, order_id, - leverage=self._leverage, + leverage=self._leverage[trading_pair], position=position_action.name)) return order_result except asyncio.CancelledError: @@ -495,7 +495,7 @@ async def _user_stream_event_listener(self): order_type=OrderType.LIMIT if order_message.get("o") == "LIMIT" else OrderType.MARKET, price=Decimal(order_message.get("L")), amount=Decimal(order_message.get("l")), - leverage=self._leverage, + leverage=self._leverage[convert_from_exchange_trading_pair(order_message.get("s"))], trade_fee=self.get_fee( base_currency=tracked_order.base_asset, quote_currency=tracked_order.quote_asset, @@ -786,7 +786,7 @@ async def _update_order_fills_from_trades(self): Decimal(trade["price"]), Decimal(trade["qty"])), exchange_trade_id=trade["id"], - leverage=self._leverage, + leverage=self._leverage[tracked_order.trading_pair], position=tracked_order.position ) ) @@ -885,8 +885,8 @@ async def _set_leverage(self, trading_pair: str, leverage: int = 1): is_signed=True ) if set_leverage["leverage"] == leverage: - self._leverage = leverage - self.logger().info(f"Leverage Successfully set to {leverage}.") + self._leverage[trading_pair] = leverage + self.logger().info(f"Leverage Successfully set to {leverage} for {trading_pair}.") else: self.logger().error("Unable to set leverage.") return leverage @@ -894,13 +894,12 @@ async def _set_leverage(self, trading_pair: str, leverage: int = 1): def set_leverage(self, trading_pair: str, leverage: int = 1): safe_ensure_future(self._set_leverage(trading_pair, leverage)) - async def _get_funding_info(self, trading_pair): + """async def _get_funding_info(self): prem_index = await self.request("/fapi/v1/premiumIndex", params={"symbol": convert_to_exchange_trading_pair(trading_pair)}) - self._funding_info = Decimal(prem_index.get("lastFundingRate", "0")) + self._funding_info = Decimal(prem_index.get("lastFundingRate", "0"))""" def get_funding_info(self, trading_pair): - safe_ensure_future(self._get_funding_info(trading_pair)) - return self._funding_info + return self._funding_info[trading_pair] async def _set_position_mode(self, position_mode: PositionMode): initial_mode = await self._get_position_mode() diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py index 7e86a486d5..e8b8716aac 100644 --- a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py @@ -23,6 +23,7 @@ BuyOrderCompletedEvent, SellOrderCompletedEvent, MarketOrderFailureEvent, + FundingPaymentCompletedEvent, OrderFilledEvent, OrderType, TradeType, @@ -274,12 +275,12 @@ async def _create_order(self, api_params = {"pair": convert_to_exchange_trading_pair(trading_pair)} if position_action == PositionAction.OPEN: api_params.update({"side": 0 if trade_type == TradeType.BUY else 1, - "margin": str(amount / self._leverage), - "leverage": self._leverage, + "margin": str(amount / self._leverage[trading_pair]), + "leverage": self._leverage[trading_pair], "minBaseAssetAmount": amount}) else: api_params.update({"minimalQuoteAsset": price * amount}) - self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, self._leverage, position_action.name) + self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, self._leverage[trading_pair], position_action.name) try: order_result = await self._api_request("post", f"perpfi/{position_action.name.lower()}", api_params) hash = order_result.get("txHash") @@ -295,7 +296,8 @@ async def _create_order(self, event_tag = MarketEvent.BuyOrderCreated if trade_type is TradeType.BUY else MarketEvent.SellOrderCreated event_class = BuyOrderCreatedEvent if trade_type is TradeType.BUY else SellOrderCreatedEvent self.trigger_event(event_tag, event_class(self.current_timestamp, OrderType.LIMIT, trading_pair, amount, - price, order_id, hash)) + price, order_id, hash, leverage=self._leverage[trading_pair], + position=position_action.name)) else: self.trigger_event(MarketEvent.OrderFailure, MarketOrderFailureEvent(self.current_timestamp, order_id, OrderType.LIMIT)) @@ -380,7 +382,9 @@ async def _update_order_status(self): Decimal(str(tracked_order.price)), Decimal(str(tracked_order.amount)), fee, - exchange_trade_id=order_id + exchange_trade_id=order_id, + leverage=self._leverage[tracked_order.trading_pair], + position=tracked_order.position ) ) tracked_order.last_state = "FILLED" @@ -518,18 +522,29 @@ async def _update_balances(self): self._in_flight_orders_snapshot_timestamp = self.current_timestamp async def _update_positions(self): - tasks = [] + position_tasks = [] + funding_payment_tasks = [] + funding_info_tasks = [] for pair in self._trading_pairs: - tasks.append(self._api_request("post", - "perpfi/position", - {"pair": convert_to_exchange_trading_pair(pair)})) - positions = await safe_gather(*tasks, return_exceptions=True) - for trading_pair, position in zip(self._trading_pairs, positions.get("position", {})): + position_tasks.append(self._api_request("post", + "perpfi/position", + {"pair": convert_to_exchange_trading_pair(pair)})) + funding_payment_tasks.append(self._api_request("get", + "perpfi/funding_payment", + {"pair": convert_to_exchange_trading_pair(pair)})) + funding_info_tasks.append(self._api_request("get", + "perpfi/funding", + {"pair": convert_to_exchange_trading_pair(pair)})) + positions = await safe_gather(*position_tasks, return_exceptions=True) + funding_payments = await safe_gather(*funding_payment_tasks, return_exceptions=True) + funding_infos = await safe_gather(*funding_info_tasks, return_exceptions=True) + for trading_pair, position in zip(self._trading_pairs, positions): + position = position.get("position", {}) position_side = PositionSide.LONG if position.get("size", 0) > 0 else PositionSide.SHORT unrealized_pnl = Decimal(position.get("pnl")) entry_price = Decimal(position.get("entryPrice")) amount = Decimal(position.get("size")) - leverage = self._leverage + leverage = self._leverage[trading_pair] if amount != 0: self._account_positions[trading_pair + position_side.name] = Position( trading_pair=trading_pair, @@ -543,6 +558,27 @@ async def _update_positions(self): if (trading_pair + position_side.name) in self._account_positions: del self._account_positions[trading_pair + position_side.name] + for trading_pair, funding_payment in zip(self._trading_pairs, funding_payments): + payment = Decimal(str(funding_payment.payment)) + action = "paid" if payment < 0 else "received" + if payment != Decimal("0"): + self.logger().info(f"Funding payment of {payment} {action} on {trading_pair} market.") + self.trigger_event(MarketEvent.FundingPaymentCompleted, + FundingPaymentCompletedEvent(timestamp=funding_payment.timestamp, + market=self.name, + rate=self._funding_info["rate"], + symbol=trading_pair, + amount=payment)) + + for trading_pair, funding_info in zip(self._trading_pairs, funding_infos): + self._funding_info[trading_pair] = funding_info["fr"] + + def get_funding_info(self, trading_pair): + return self._funding_info[trading_pair] + + def set_leverage(self, trading_pair: str, leverage: int = 1): + self._leverage[trading_pair] = leverage + async def _http_client(self) -> aiohttp.ClientSession: """ :returns Shared client session instance diff --git a/hummingbot/core/event/events.py b/hummingbot/core/event/events.py index a085e2ba32..c4a02d1e76 100644 --- a/hummingbot/core/event/events.py +++ b/hummingbot/core/event/events.py @@ -34,6 +34,7 @@ class MarketEvent(Enum): TransactionFailure = 199 BuyOrderCreated = 200 SellOrderCreated = 201 + FundingPaymentCompleted = 202 class NewBlocksWatcherEvent(Enum): diff --git a/hummingbot/strategy/strategy_base.pxd b/hummingbot/strategy/strategy_base.pxd index 2ed151b74e..efb35d8b09 100644 --- a/hummingbot/strategy/strategy_base.pxd +++ b/hummingbot/strategy/strategy_base.pxd @@ -29,6 +29,7 @@ cdef class StrategyBase(TimeIterator): cdef c_did_expire_order(self, object expired_event) cdef c_did_complete_buy_order(self, object order_completed_event) cdef c_did_complete_sell_order(self, object order_completed_event) + cdef c_did_complete_funding_payment(self, object funding_payment_completed_event) cdef c_did_fail_order_tracker(self, object order_failed_event) cdef c_did_cancel_order_tracker(self, object order_cancelled_event) diff --git a/hummingbot/strategy/strategy_base.pyx b/hummingbot/strategy/strategy_base.pyx index 4bc661693c..d214b288e8 100755 --- a/hummingbot/strategy/strategy_base.pyx +++ b/hummingbot/strategy/strategy_base.pyx @@ -46,6 +46,11 @@ cdef class SellOrderCompletedListener(BaseStrategyEventListener): self._owner.c_did_complete_sell_order_tracker(arg) +cdef class FundingPaymentCompletedListener(BaseStrategyEventListener): + cdef c_call(self, object arg): + self._owner.c_did_complete_funding_payment(arg) + + cdef class OrderFilledListener(BaseStrategyEventListener): cdef c_call(self, object arg): self._owner.c_did_fill_order(arg) @@ -83,6 +88,7 @@ cdef class SellOrderCreatedListener(BaseStrategyEventListener): cdef class StrategyBase(TimeIterator): BUY_ORDER_COMPLETED_EVENT_TAG = MarketEvent.BuyOrderCompleted.value SELL_ORDER_COMPLETED_EVENT_TAG = MarketEvent.SellOrderCompleted.value + FUNDING_PAYMENT_COMPLETED_EVENT_TAG = MarketEvent.FundingPaymentCompleted.value ORDER_FILLED_EVENT_TAG = MarketEvent.OrderFilled.value ORDER_CANCELLED_EVENT_TAG = MarketEvent.OrderCancelled.value ORDER_EXPIRED_EVENT_TAG = MarketEvent.OrderExpired.value @@ -105,6 +111,7 @@ cdef class StrategyBase(TimeIterator): self._sb_expire_order_listener = OrderExpiredListener(self) self._sb_complete_buy_order_listener = BuyOrderCompletedListener(self) self._sb_complete_sell_order_listener = SellOrderCompletedListener(self) + self._sb_complete_funding_payment_listener = FundingPaymentCompletedListener(self) self._sb_delegate_lock = False @@ -255,6 +262,7 @@ cdef class StrategyBase(TimeIterator): typed_market.c_add_listener(self.ORDER_EXPIRED_EVENT_TAG, self._sb_expire_order_listener) typed_market.c_add_listener(self.BUY_ORDER_COMPLETED_EVENT_TAG, self._sb_complete_buy_order_listener) typed_market.c_add_listener(self.SELL_ORDER_COMPLETED_EVENT_TAG, self._sb_complete_sell_order_listener) + typed_market.c_add_listener(self.FUNDING_PAYMENT_COMPLETED_EVENT_TAG, self._sb_complete_funding_payment_listener) self._sb_markets.add(typed_market) cdef c_remove_markets(self, list markets): @@ -273,6 +281,7 @@ cdef class StrategyBase(TimeIterator): typed_market.c_remove_listener(self.ORDER_EXPIRED_EVENT_TAG, self._sb_expire_order_listener) typed_market.c_remove_listener(self.BUY_ORDER_COMPLETED_EVENT_TAG, self._sb_complete_buy_order_listener) typed_market.c_remove_listener(self.SELL_ORDER_COMPLETED_EVENT_TAG, self._sb_complete_sell_order_listener) + typed_market.c_remove_listener(self.FUNDING_PAYMENT_COMPLETED_EVENT_TAG, self._sb_complete_funding_payment_listener) self._sb_markets.remove(typed_market) cdef object c_sum_flat_fees(self, str quote_asset, list flat_fees): @@ -317,6 +326,9 @@ cdef class StrategyBase(TimeIterator): cdef c_did_complete_sell_order(self, object order_completed_event): pass + + cdef c_did_complete_funding_payment(self, object funding_payment_completed_event): + pass # ---------------------------------------------------------------------------------------------------------- # diff --git a/hummingbot/strategy/strategy_py_base.pyx b/hummingbot/strategy/strategy_py_base.pyx index 6d846e5eab..9b8546a8cb 100644 --- a/hummingbot/strategy/strategy_py_base.pyx +++ b/hummingbot/strategy/strategy_py_base.pyx @@ -89,3 +89,9 @@ cdef class StrategyPyBase(StrategyBase): def did_complete_sell_order(self, order_completed_event): pass + + cdef c_did_complete_funding_payment(self, object funding_payment_completed_event): + self.did_complete_funding_payment(funding_payment_completed_event) + + def did_complete_funding_payment(self, funding_payment_completed_event): + pass From bf2302a2bc3bf9528bfd76148261283a08d78e9d Mon Sep 17 00:00:00 2001 From: vic-en Date: Wed, 17 Feb 2021 20:30:24 +0100 Subject: [PATCH 24/73] (feat) update binance perp connector --- .../binance_perpetual_derivative.py | 102 ++++++++++++++---- .../perpetual_finance_derivative.py | 2 +- hummingbot/connector/derivative_base.py | 4 +- 3 files changed, 86 insertions(+), 22 deletions(-) diff --git a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py index ecb6d777a3..f2d4359d35 100644 --- a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py +++ b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py @@ -21,6 +21,9 @@ import hmac import time import logging +import ujson +import websockets +from websockets.exceptions import ConnectionClosed from decimal import Decimal from typing import Optional, List, Dict, Any, AsyncIterable from urllib.parse import urlencode @@ -37,6 +40,7 @@ BuyOrderCompletedEvent, BuyOrderCreatedEvent, SellOrderCreatedEvent, + FundingPaymentCompletedEvent, OrderFilledEvent, SellOrderCompletedEvent, PositionSide, PositionMode, PositionAction) from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather @@ -86,6 +90,7 @@ class BinancePerpetualDerivative(DerivativeBase): MARKET_ORDER_FILLED_EVENT_TAG = MarketEvent.OrderFilled MARKET_BUY_ORDER_CREATED_EVENT_TAG = MarketEvent.BuyOrderCreated MARKET_SELL_ORDER_CREATED_EVENT_TAG = MarketEvent.SellOrderCreated + MARKET_FUNDING_PAYMENT_COMPLETED_EVENT_TAG = MarketEvent.FundingPaymentCompleted API_CALL_TIMEOUT = 10.0 SHORT_POLL_INTERVAL = 5.0 @@ -129,6 +134,7 @@ def __init__(self, self._status_polling_task = None self._user_stream_event_listener_task = None self._trading_rules_polling_task = None + self._funding_info_polling_task = None self._last_poll_timestamp = 0 self._throttler = Throttler((10.0, 1.0)) self._funding_payment_span = [0, 15] @@ -173,6 +179,7 @@ def stop(self, clock: Clock): async def start_network(self): self._order_book_tracker.start() self._trading_rules_polling_task = safe_ensure_future(self._trading_rules_polling_loop()) + self._funding_info_polling_task = safe_ensure_future(self._funding_info_polling_loop()) if self._trading_required: self._status_polling_task = safe_ensure_future(self._status_polling_loop()) self._user_stream_tracker_task = safe_ensure_future(self._user_stream_tracker.start()) @@ -188,8 +195,10 @@ def _stop_network(self): self._user_stream_event_listener_task.cancel() if self._trading_rules_polling_task is not None: self._trading_rules_polling_task.cancel() + if self._funding_info_polling_task is not None: + self._funding_info_polling_task.cancel() self._status_polling_task = self._user_stream_tracker_task = \ - self._user_stream_event_listener_task = None + self._user_stream_event_listener_task = self._funding_info_polling_task = None async def stop_network(self): self._stop_network() @@ -548,22 +557,26 @@ async def _user_stream_event_listener(self): self.stop_tracking_order(tracked_order.client_order_id) elif event_type == "ACCOUNT_UPDATE": update_data = event_message.get("a", {}) - # update balances - for asset in update_data.get("B", []): - asset_name = asset["a"] - self._account_balances[asset_name] = Decimal(asset["wb"]) - self._account_available_balances[asset_name] = Decimal(asset["cw"]) - - # update position - for asset in update_data.get("P", []): - position = self._account_positions.get(f"{asset['s']}{asset['ps']}", None) - if position is not None: - position.update_position(position_side=PositionSide[asset["ps"]], - unrealized_pnl = Decimal(asset["up"]), - entry_price = Decimal(asset["ep"]), - amount = Decimal(asset["pa"])) - else: - await self._update_positions() + event_reason = update_data.get("m", {}) + if event_reason == "FUNDING_FEE": + await self.get_funding_payment(event_message.get("E", int(time.time()))) + else: + # update balances + for asset in update_data.get("B", []): + asset_name = asset["a"] + self._account_balances[asset_name] = Decimal(asset["wb"]) + self._account_available_balances[asset_name] = Decimal(asset["cw"]) + + # update position + for asset in update_data.get("P", []): + position = self._account_positions.get(f"{asset['s']}{asset['ps']}", None) + if position is not None: + position.update_position(position_side=PositionSide[asset["ps"]], + unrealized_pnl = Decimal(asset["up"]), + entry_price = Decimal(asset["ep"]), + amount = Decimal(asset["pa"])) + else: + await self._update_positions() elif event_type == "MARGIN_CALL": positions = event_message.get("p", []) total_maint_margin_required = 0 @@ -674,6 +687,37 @@ async def _trading_rules_polling_loop(self): "Check network connection.") await asyncio.sleep(0.5) + async def _funding_info_polling_loop(self): + while True: + try: + ws_subscription_path: str = "/".join([f"{convert_to_exchange_trading_pair(trading_pair).lower()}@markPrice" + for trading_pair in self._trading_pairs]) + stream_url: str = f"{self._stream_url}?streams={ws_subscription_path}" + async with websockets.connect(stream_url) as ws: + ws: websockets.WebSocketClientProtocol = ws + try: + while True: + try: + raw_msg: str = await asyncio.wait_for(ws.recv(), timeout=10.0) + msg = ujson.loads(raw_msg) + trading_pair = msg["s"] + self._funding_info[trading_pair] = {"indexPrice": msg["i"], + "markPrice": msg["p"], + "nextFundingTime": msg["T"], + "rate": msg["r"]} + except asyncio.TimeoutError: + await ws.pong(data=b'') + except ConnectionClosed: + continue + finally: + await ws.close() + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error updating funding info. Retrying after 10 seconds... ", + exc_info=True) + await asyncio.sleep(10.0) + async def _status_polling_loop(self): while True: try: @@ -894,9 +938,27 @@ async def _set_leverage(self, trading_pair: str, leverage: int = 1): def set_leverage(self, trading_pair: str, leverage: int = 1): safe_ensure_future(self._set_leverage(trading_pair, leverage)) - """async def _get_funding_info(self): - prem_index = await self.request("/fapi/v1/premiumIndex", params={"symbol": convert_to_exchange_trading_pair(trading_pair)}) - self._funding_info = Decimal(prem_index.get("lastFundingRate", "0"))""" + async def get_funding_payment(self): + funding_payment_tasks = [] + for pair in self._trading_pairs: + funding_payment_tasks.append(self.request(path="/fapi/v1/income", + params={"symbol": convert_to_exchange_trading_pair(pair), "incomeType": "FUNDING_FEE", "limit": 1}, + method=MethodType.POST, + add_timestamp=True, + is_signed=True)) + funding_payments = await safe_gather(*funding_payment_tasks, return_exceptions=True) + for funding_payment in funding_payments: + payment = Decimal(funding_payment["income"]) + action = "paid" if payment < 0 else "received" + trading_pair = convert_to_exchange_trading_pair(funding_payment["symbol"]) + if payment != Decimal("0"): + self.logger().info(f"Funding payment of {payment} {action} on {trading_pair} market.") + self.trigger_event(self.MARKET_FUNDING_PAYMENT_COMPLETED_EVENT_TAG, + FundingPaymentCompletedEvent(timestamp=funding_payment["time"], + market=self.name, + rate=self._funding_info[trading_pair]["rate"], + symbol=trading_pair, + amount=payment)) def get_funding_info(self, trading_pair): return self._funding_info[trading_pair] diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py index e8b8716aac..71bb2ffad7 100644 --- a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py @@ -566,7 +566,7 @@ async def _update_positions(self): self.trigger_event(MarketEvent.FundingPaymentCompleted, FundingPaymentCompletedEvent(timestamp=funding_payment.timestamp, market=self.name, - rate=self._funding_info["rate"], + rate=self._funding_info[trading_pair]["rate"], symbol=trading_pair, amount=payment)) diff --git a/hummingbot/connector/derivative_base.py b/hummingbot/connector/derivative_base.py index ff4750017a..5bd2667e33 100644 --- a/hummingbot/connector/derivative_base.py +++ b/hummingbot/connector/derivative_base.py @@ -50,10 +50,12 @@ def supported_position_modes(self): def get_funding_info(self, trading_pair): """ - return a dictionary containing: + return a dictionary as follows: + self._trading_info[trading_pair] = { "indexPrice": (i.e "21.169488483519444444") "markPrice": price used for both pnl on most derivatives (i.e "21.210103847902463671") "nextFundingTime": next funding time in unix timestamp (i.e "1612780270") "rate": next funding rate as a decimal and not percentage (i.e 0.00007994084744229488) + } """ raise NotImplementedError From 82c90ee57147c636099a75dbd8f4748ac9c4231a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dennis=20Oca=C3=B1a?= <50150287+dennisocana@users.noreply.github.com> Date: Thu, 18 Feb 2021 14:35:26 +0800 Subject: [PATCH 25/73] (feat) delete exchange connector request template --- .github/ISSUE_TEMPLATE/exchange_request.md | 49 ---------------------- 1 file changed, 49 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/exchange_request.md diff --git a/.github/ISSUE_TEMPLATE/exchange_request.md b/.github/ISSUE_TEMPLATE/exchange_request.md deleted file mode 100644 index 51c2cc7308..0000000000 --- a/.github/ISSUE_TEMPLATE/exchange_request.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -name: Exchange connector request -about: Suggest a new exchange connector for hummingbot -title: '' -labels: new exchange -assignees: '' - ---- - -## Exchange Details ✏️ - -- **Name of exchange**: -- **Exchange URL**: -- **Link to API docs**: -- **Type of exchange**: [ ] Centralized [ ] Decentralized -- **Requester details**: Are you affiliated with the exchange? [ ] yes [ ] no - - If yes, what is your role? - -## Rationale / impact ✏️ -(Describe your rationale for building this exchange connector, impact for hummingbot users/community) - -## Additional information ✏️ -(Provide any other useful information that may be helpful to know about this exchange) - ---- - -⚠️ ***Note: do not modify below here*** - -## Developer notes - -This feature request entails building a new exchange connector to allow Hummingbot to connect to an exchange that is currently not supported. - -### Resources -- [Exchange connector developer guide](https://docs.hummingbot.io/developers/connectors/) -- [Discord forum](https://discord.hummingbot.io) - -### Deliverables -1. A complete set of exchange connector files as listed [above](#developer-notes-resources). -2. Unit tests (see [existing unit tests](https://github.com/CoinAlpha/hummingbot/tree/master/test/integration)): - 1. Exchange market test ([example](https://github.com/CoinAlpha/hummingbot/blob/master/test/integration/test_binance_market.py)) - 2. Order book tracker ([example](https://github.com/CoinAlpha/hummingbot/blob/master/test/integration/test_binance_order_book_tracker.py)) - 3. User stream tracker ([example](https://github.com/CoinAlpha/hummingbot/blob/master/test/integration/test_binance_user_stream_tracker.py)) -3. Documentation: - 1. Code commenting (particularly for any code that is materially different from the templates/examples) - 2. Any specific instructions for the use of that exchange connector ([example](https://docs.hummingbot.io/connectors/binance/)) - -### Required skills -- Python -- Previous Cython experience is a plus (optional) \ No newline at end of file From 8e2bbad0191af04b5488148910f39967dad1c2a7 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Thu, 18 Feb 2021 14:55:31 +0800 Subject: [PATCH 26/73] (refactor) ProbitAuth module and other Exchange methods --- .../probit_api_order_book_data_source.py | 2 +- .../connector/exchange/probit/probit_auth.py | 56 ++++++------------ .../exchange/probit/probit_constants.py | 1 + .../exchange/probit/probit_exchange.py | 58 ++++++++++++++++--- 4 files changed, 70 insertions(+), 47 deletions(-) diff --git a/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py b/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py index 005600c462..1d448bbb71 100644 --- a/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py @@ -61,7 +61,7 @@ async def fetch_trading_pairs() -> List[str]: async with client.get(f"{constants.MARKETS_URL}") as response: if response.status == 200: resp_json: Dict[str, Any] = await response.json() - return [market["market_id"] for market in resp_json["data"]] + return [market["id"] for market in resp_json["data"]] return [] @staticmethod diff --git a/hummingbot/connector/exchange/probit/probit_auth.py b/hummingbot/connector/exchange/probit/probit_auth.py index d3a004f7fa..53ec139f36 100644 --- a/hummingbot/connector/exchange/probit/probit_auth.py +++ b/hummingbot/connector/exchange/probit/probit_auth.py @@ -1,11 +1,7 @@ #!/usr/bin/env python -import aiohttp import base64 import time -import ujson - -import hummingbot.connector.exchange.probit.probit_constants as constants from typing import Dict, Any @@ -20,49 +16,32 @@ def __init__(self, api_key: str, secret_key: str): self.secret_key: str = secret_key self._oauth_token: str = None self._oauth_token_expiration_time: int = -1 - self._http_client: aiohttp.ClientSession = aiohttp.ClientSession() - def _token_has_expired(self): + @property + def oauth_token(self): + return self._oauth_token + + @property + def token_payload(self): + payload = f"{self.api_key}:{self.secret_key}".encode() + return base64.b64encode(payload).decode() + + def token_has_expired(self): now: int = int(time.time()) return now >= self._oauth_token_expiration_time - def _update_expiration_time(self, expiration_time: int): - self._oauth_token_expiration_time = expiration_time + def update_oauth_token(self, new_token: str): + self._oauth_token = new_token - async def _generate_oauth_token(self) -> str: - try: - now: int = int(time.time()) - headers: Dict[str, Any] = self.get_headers() - payload = f"{self.api_key}:{self.secret_key}".encode() - b64_payload = base64.b64encode(payload).decode() - headers.update({ - "Authorization": f"Basic {b64_payload}" - }) - body = ujson.dumps({ - "grant_type": "client_credentials" - }) - resp = await self._http_client.post(url=constants.TOKEN_URL, - headers=headers, - data=body) - token_resp = await resp.json() - - if resp.status != 200: - raise ValueError(f"{__name__}: Error occurred retrieving new OAuth Token. Response: {token_resp}") - - # POST /token endpoint returns both access_token and expires_in - # Updates _oauth_token_expiration_time - - self._update_expiration_time(now + token_resp["expires_in"]) - return token_resp["access_token"] - except Exception as e: - raise e + def update_expiration_time(self, expiration_time: int): + self._oauth_token_expiration_time = expiration_time async def get_oauth_token(self) -> str: if self._oauth_token is None or self._token_has_expired(): - self._oauth_token = await self._generate_oauth_token() + self._oauth_token = await self.generate_oauth_token() return self._oauth_token - async def generate_auth_dict(self): + def generate_auth_dict(self): """ Generates authentication signature and return it in a dictionary along with other inputs :return: a dictionary of request info including the request signature @@ -70,9 +49,8 @@ async def generate_auth_dict(self): headers = self.get_headers() - access_token = await self.get_oauth_token() headers.update({ - "Authorization": f"Bearer {access_token}" + "Authorization": f"Bearer {self._oauth_token}" }) return headers diff --git a/hummingbot/connector/exchange/probit/probit_constants.py b/hummingbot/connector/exchange/probit/probit_constants.py index 95607932a6..6583e26a5b 100644 --- a/hummingbot/connector/exchange/probit/probit_constants.py +++ b/hummingbot/connector/exchange/probit/probit_constants.py @@ -8,6 +8,7 @@ REST_API_VERSON = "v1" # REST API Public Endpoints +TIME_URL = f"{REST_URL+REST_API_VERSON}/time" TICKER_URL = f"{REST_URL+REST_API_VERSON}/ticker" MARKETS_URL = f"{REST_URL+REST_API_VERSON}/market" ORDER_BOOK_URL = f"{REST_URL+REST_API_VERSON}/order_book" diff --git a/hummingbot/connector/exchange/probit/probit_exchange.py b/hummingbot/connector/exchange/probit/probit_exchange.py index bf7fc2c977..11ed57901e 100644 --- a/hummingbot/connector/exchange/probit/probit_exchange.py +++ b/hummingbot/connector/exchange/probit/probit_exchange.py @@ -44,6 +44,7 @@ from hummingbot.core.network_iterator import NetworkStatus from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather from hummingbot.logger import HummingbotLogger +import ujson probit_logger = None s_decimal_NaN = Decimal("nan") @@ -222,7 +223,12 @@ async def check_network(self) -> NetworkStatus: """ try: # since there is no ping endpoint, the lowest rate call is to get BTC-USDT ticker - await self._api_request("get", "public/get-ticker?instrument_name=BTC_USDT") + resp = await self._api_request( + method="GET", + path_url=CONSTANTS.TIME_URL + ) + if resp.status != 200: + raise except asyncio.CancelledError: raise except Exception: @@ -255,7 +261,10 @@ async def _trading_rules_polling_loop(self): await asyncio.sleep(0.5) async def _update_trading_rules(self): - market_info = await self._api_request("GET", path_url="public/get-instruments") + market_info = await self._api_request( + method="GET", + path_url=CONSTANTS.MARKETS_URL + ) self._trading_rules.clear() self._trading_rules = self._format_trading_rules(market_info) @@ -294,14 +303,44 @@ def _format_trading_rules(self, market_info: Dict[str, Any]) -> Dict[str, Tradin quantity_step = Decimal("1") / Decimal(str(math.pow(10, quantity_decimals))) result[trading_pair] = TradingRule(trading_pair=trading_pair, - min_order_size=Decimal(str(market["min_cost"])), - max_order_size=Decimal(str(market["max_cost"])), + min_order_size=Decimal(str(market["min_quantity"])), + max_order_size=Decimal(str(market["max_quantity"])), + min_order_value=Decimal(str(market["min_cost"])), min_price_increment=Decimal(str(market["price_increment"])), min_base_amount_increment=quantity_step) except Exception: self.logger().error(f"Error parsing the trading pair rule {market}. Skipping.", exc_info=True) return result + async def _get_auth_headers(self, http_client: aiohttp.ClientSession) -> Dict[str, Any]: + if self._probit_auth.token_has_expired: + try: + now: int = int(time.time()) + headers = self._probit_auth.get_headers() + headers.update({ + "Authorization": f"Basic {self._probit_auth.token_payload}" + }) + body = ujson.dumps({ + "grant_type": "client_credentials" + }) + resp = await http_client.post(url=CONSTANTS.TOKEN_URL, + headers=headers, + data=body) + token_resp = await resp.json() + + if resp.status != 200: + raise ValueError(f"Error occurred retrieving new OAuth Token. Response: {token_resp}") + + # POST /token endpoint returns both access_token and expires_in + # Updates _oauth_token_expiration_time + + self._probit_auth.update_expiration_time(now + token_resp["expires_in"]) + self._probit_auth.update_oauth_token(token_resp["access_token"]) + except Exception as e: + raise e + + return self._probit_auth.generate_auth_dict() + async def _api_request(self, method: str, path_url: str, @@ -319,7 +358,7 @@ async def _api_request(self, client = await self._http_client() if is_auth_required: - headers = self._probit_auth.generate_auth_dict() + headers = await self._get_auth_headers(client) else: headers = self._probit_auth.get_headers() @@ -337,8 +376,7 @@ async def _api_request(self, if response.status != 200: raise IOError(f"Error fetching data from {path_url}. HTTP status is {response.status}. " f"Message: {parsed_response}") - if parsed_response["code"] != 0: - raise IOError(f"{path_url} API call failed, response: {parsed_response}") + return parsed_response def get_order_price_quantum(self, trading_pair: str, price: Decimal): @@ -422,10 +460,16 @@ async def _create_order(self, amount = self.quantize_order_amount(trading_pair, amount) price = self.quantize_order_price(trading_pair, price) + if amount < trading_rule.min_order_size: raise ValueError(f"{trade_type.name} order amount {amount} is lower than the minimum order size " f"{trading_rule.min_order_size}.") + order_value: Decimal = amount * price + if order_value < trading_rule.min_order_value: + raise ValueError(f"{trade_type.name} order value {order_value} is lower than the minimum order value " + f"{trading_rule.min_order_value}") + body_params = { "market_id": trading_pair, "type": "limit", # ProBit Order Types ["limit", "market"} From 30add68fe6dce32fe07eb4e480425c6d1dbc7ee4 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Thu, 18 Feb 2021 17:29:17 +0800 Subject: [PATCH 27/73] (fix) outstanding issues with ProbitExchange --- .../probit_api_order_book_data_source.py | 20 ++++++----- .../probit_api_user_stream_data_source.py | 4 +-- .../connector/exchange/probit/probit_auth.py | 36 ++++++++++++++++--- .../exchange/probit/probit_exchange.py | 12 ++++--- .../connector/exchange/probit/probit_utils.py | 2 +- 5 files changed, 54 insertions(+), 20 deletions(-) diff --git a/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py b/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py index 1d448bbb71..3f04d83710 100644 --- a/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py @@ -51,8 +51,10 @@ async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, flo if response.status == 200: resp_json = await response.json() if "data" in resp_json: - for trading_pair in resp_json["data"]: - result[trading_pair["market_id"]] = trading_pair["last"] + for market in resp_json["data"]: + if market["market_id"] in trading_pairs: + result[market["market_id"]] = float(market["last"]) + return result @staticmethod @@ -147,11 +149,11 @@ async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asynci if "reset" in msg and msg["reset"] is True: # Ignores first response from "recent_trades" channel. This response details the last 100 trades. continue - for trade_entry in msg["recent_trades"]: trade_msg: OrderBookMessage = ProbitOrderBook.trade_message_from_exchange( msg=trade_entry, - timestamp=msg_timestamp) + timestamp=msg_timestamp, + metadata={"market_id": msg["market_id"]}) output.put_nowait(trade_msg) except asyncio.CancelledError: raise @@ -191,10 +193,12 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp timestamp=msg_timestamp, ) output.put_nowait(snapshot_msg) - continue - for diff_entry in msg["order_books"]: - diff_msg: OrderBookMessage = ProbitOrderBook.diff_message_from_exchange(diff_entry, - msg_timestamp) + else: + diff_msg: OrderBookMessage = ProbitOrderBook.diff_message_from_exchange( + msg=msg, + timestamp=msg_timestamp, + metadata={"market_id": msg["market_id"]} + ) output.put_nowait(diff_msg) except asyncio.CancelledError: raise diff --git a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py index 9678c8fde2..fcd3f985a6 100644 --- a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py @@ -61,10 +61,10 @@ async def _authenticate(self, ws: websockets.WebSocketClientProtocol): Authenticates user to websocket """ try: - access_token: str = await self._probit_auth.get_oauth_token() + await self._probit_auth.get_auth_headers() auth_payload: Dict[str, Any] = { "type": "authorization", - "token": access_token + "token": self._probit_auth.oauth_token } await ws.send(ujson.dumps(auth_payload)) auth_resp = await ws.recv() diff --git a/hummingbot/connector/exchange/probit/probit_auth.py b/hummingbot/connector/exchange/probit/probit_auth.py index 53ec139f36..51a93882d9 100644 --- a/hummingbot/connector/exchange/probit/probit_auth.py +++ b/hummingbot/connector/exchange/probit/probit_auth.py @@ -1,7 +1,11 @@ #!/usr/bin/env python +import aiohttp import base64 import time +import ujson + +import hummingbot.connector.exchange.probit.probit_constants as CONSTANTS from typing import Dict, Any @@ -36,10 +40,34 @@ def update_oauth_token(self, new_token: str): def update_expiration_time(self, expiration_time: int): self._oauth_token_expiration_time = expiration_time - async def get_oauth_token(self) -> str: - if self._oauth_token is None or self._token_has_expired(): - self._oauth_token = await self.generate_oauth_token() - return self._oauth_token + async def get_auth_headers(self, http_client: aiohttp.ClientSession = aiohttp.ClientSession()) -> Dict[str, Any]: + if self.token_has_expired: + try: + now: int = int(time.time()) + headers = self.get_headers() + headers.update({ + "Authorization": f"Basic {self.token_payload}" + }) + body = ujson.dumps({ + "grant_type": "client_credentials" + }) + resp = await http_client.post(url=CONSTANTS.TOKEN_URL, + headers=headers, + data=body) + token_resp = await resp.json() + + if resp.status != 200: + raise ValueError(f"Error occurred retrieving new OAuth Token. Response: {token_resp}") + + # POST /token endpoint returns both access_token and expires_in + # Updates _oauth_token_expiration_time + + self.update_expiration_time(now + token_resp["expires_in"]) + self.update_oauth_token(token_resp["access_token"]) + except Exception as e: + raise e + + return self.generate_auth_dict() def generate_auth_dict(self): """ diff --git a/hummingbot/connector/exchange/probit/probit_exchange.py b/hummingbot/connector/exchange/probit/probit_exchange.py index 11ed57901e..864f5fd5c8 100644 --- a/hummingbot/connector/exchange/probit/probit_exchange.py +++ b/hummingbot/connector/exchange/probit/probit_exchange.py @@ -92,10 +92,12 @@ def __init__(self, self._in_flight_orders = {} # Dict[client_order_id:str, ProbitInFlightOrder] self._order_not_found_records = {} # Dict[client_order_id:str, count:int] self._trading_rules = {} # Dict[trading_pair:str, TradingRule] + self._last_poll_timestamp = 0 + self._status_polling_task = None + self._user_stream_tracker_task = None self._user_stream_event_listener_task = None self._trading_rules_polling_task = None - self._last_poll_timestamp = 0 @property def name(self) -> str: @@ -192,7 +194,7 @@ async def start_network(self): self._trading_rules_polling_task = safe_ensure_future(self._trading_rules_polling_loop()) if self._trading_required: self._status_polling_task = safe_ensure_future(self._status_polling_loop()) - self._user_stream_tracker_task = safe_ensure_future(self._user_stream_tracker.start()) + # self._user_stream_tracker_task = safe_ensure_future(self._user_stream_tracker.start()) self._user_stream_event_listener_task = safe_ensure_future(self._user_stream_event_listener()) async def stop_network(self): @@ -227,7 +229,7 @@ async def check_network(self) -> NetworkStatus: method="GET", path_url=CONSTANTS.TIME_URL ) - if resp.status != 200: + if "data" not in resp: raise except asyncio.CancelledError: raise @@ -358,7 +360,7 @@ async def _api_request(self, client = await self._http_client() if is_auth_required: - headers = await self._get_auth_headers(client) + headers = await self._probit_auth.get_auth_headers(client) else: headers = self._probit_auth.get_headers() @@ -818,7 +820,7 @@ async def get_open_orders(self) -> List[OpenOrder]: self.logger().info(f"Unexpected response from GET {CONSTANTS.OPEN_ORDER_URL}. " f"Params: {query_params} " f"Response: {result} ") - for order in result["data"]["order_list"]: + for order in result["data"]: if order["type"] != "limit": raise Exception(f"Unsupported order type {order['type']}") ret_val.append( diff --git a/hummingbot/connector/exchange/probit/probit_utils.py b/hummingbot/connector/exchange/probit/probit_utils.py index 462a162f93..5920ef47d7 100644 --- a/hummingbot/connector/exchange/probit/probit_utils.py +++ b/hummingbot/connector/exchange/probit/probit_utils.py @@ -63,7 +63,7 @@ def convert_diff_message_to_order_book_row(message: OrderBookMessage) -> Tuple[L asks = [] for entry in data: - order_row = OrderBookRow(entry["price"], entry["quantity"], update_id) + order_row = OrderBookRow(float(entry["price"]), float(entry["quantity"]), update_id) if entry["side"] == "buy": bids.append(order_row) elif entry["side"] == "sell": From ea7801ffee63a473f374aebd60d4f7a679733301 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Thu, 18 Feb 2021 20:24:21 +0800 Subject: [PATCH 28/73] (refactor) refactor ProbitAuth and start _user_stream_tracker --- hummingbot/connector/exchange/probit/probit_auth.py | 1 + hummingbot/connector/exchange/probit/probit_exchange.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hummingbot/connector/exchange/probit/probit_auth.py b/hummingbot/connector/exchange/probit/probit_auth.py index 51a93882d9..5401f14c94 100644 --- a/hummingbot/connector/exchange/probit/probit_auth.py +++ b/hummingbot/connector/exchange/probit/probit_auth.py @@ -30,6 +30,7 @@ def token_payload(self): payload = f"{self.api_key}:{self.secret_key}".encode() return base64.b64encode(payload).decode() + @property def token_has_expired(self): now: int = int(time.time()) return now >= self._oauth_token_expiration_time diff --git a/hummingbot/connector/exchange/probit/probit_exchange.py b/hummingbot/connector/exchange/probit/probit_exchange.py index 864f5fd5c8..e2a6b4744c 100644 --- a/hummingbot/connector/exchange/probit/probit_exchange.py +++ b/hummingbot/connector/exchange/probit/probit_exchange.py @@ -194,7 +194,7 @@ async def start_network(self): self._trading_rules_polling_task = safe_ensure_future(self._trading_rules_polling_loop()) if self._trading_required: self._status_polling_task = safe_ensure_future(self._status_polling_loop()) - # self._user_stream_tracker_task = safe_ensure_future(self._user_stream_tracker.start()) + self._user_stream_tracker_task = safe_ensure_future(self._user_stream_tracker.start()) self._user_stream_event_listener_task = safe_ensure_future(self._user_stream_event_listener()) async def stop_network(self): From 61c003ada83949156a11df3178a8a827cc156a2a Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Thu, 18 Feb 2021 20:55:33 +0800 Subject: [PATCH 29/73] (fix) fix market not starting --- .../probit_api_user_stream_data_source.py | 20 ++++++------------- .../exchange/probit/probit_exchange.py | 4 ++-- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py index fcd3f985a6..d88025528b 100644 --- a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py @@ -2,6 +2,7 @@ import asyncio import logging +import time import ujson import websockets @@ -66,9 +67,9 @@ async def _authenticate(self, ws: websockets.WebSocketClientProtocol): "type": "authorization", "token": self._probit_auth.oauth_token } - await ws.send(ujson.dumps(auth_payload)) + await ws.send(ujson.dumps(auth_payload, escape_forward_slashes=False)) auth_resp = await ws.recv() - auth_resp: Dict[str, Any] = ujson.loads(auth_resp, escape_forward_slashes=False) + auth_resp: Dict[str, Any] = ujson.loads(auth_resp) if auth_resp["result"] != "ok": self.logger().error(f"Response: {auth_resp}", @@ -92,21 +93,11 @@ async def _subscribe_to_channels(self, ws: websockets.WebSocketClientProtocol): "channel": channel } await ws.send(ujson.dumps(sub_payload)) - sub_resp = await ws.recv() - sub_resp: Dict[str, Any] = ujson.loads(sub_resp) - - if "reset" in sub_resp and sub_resp["reset"] is True: - continue - else: - self.logger().error(f"Error occured subscribing to {channel}...") - raise except asyncio.CancelledError: raise except Exception: - self.logger().error(f"Error occured subscribing to {CONSTANTS.EXCHANGE_NAME} private channels. " - f"Payload: {sub_payload} " - f"Resp: {sub_resp}", + self.logger().error(f"Error occured subscribing to {CONSTANTS.EXCHANGE_NAME} private channels. ", exc_info=True) async def _inner_messages(self, @@ -114,6 +105,7 @@ async def _inner_messages(self, try: while True: msg: str = await ws.recv() + self._last_recv_time = int(time.time()) yield msg except websockets.exceptions.ConnectionClosed: return @@ -138,7 +130,7 @@ async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: a self.logger().info("Successfully subscribed to all Private channels.") async for msg in self._inner_messages(ws): - output.put_nowait(msg) + output.put_nowait(ujson.loads(msg)) except asyncio.CancelledError: raise except Exception: diff --git a/hummingbot/connector/exchange/probit/probit_exchange.py b/hummingbot/connector/exchange/probit/probit_exchange.py index e2a6b4744c..edbb4b334f 100644 --- a/hummingbot/connector/exchange/probit/probit_exchange.py +++ b/hummingbot/connector/exchange/probit/probit_exchange.py @@ -5,6 +5,7 @@ import logging import math import time +import ujson from decimal import Decimal from typing import ( @@ -44,7 +45,6 @@ from hummingbot.core.network_iterator import NetworkStatus from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather from hummingbot.logger import HummingbotLogger -import ujson probit_logger = None s_decimal_NaN = Decimal("nan") @@ -936,7 +936,7 @@ async def _user_stream_event_listener(self): """ async for event_message in self._iter_user_event_queue(): try: - if "channel" not in event_message or event_message["channel"] not in ["open_order", "order_history", "balance", "trade_history"]: + if "channel" not in event_message and event_message["channel"] not in CONSTANTS.WS_PRIVATE_CHANNELS: continue channel = event_message["channel"] From 6e2c631aafe9f5cc931986b2484c57b9847be29b Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Thu, 18 Feb 2021 21:01:22 +0800 Subject: [PATCH 30/73] (fix) fix _api_request not sending body params correctly --- hummingbot/connector/exchange/probit/probit_exchange.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hummingbot/connector/exchange/probit/probit_exchange.py b/hummingbot/connector/exchange/probit/probit_exchange.py index edbb4b334f..c05d6cf996 100644 --- a/hummingbot/connector/exchange/probit/probit_exchange.py +++ b/hummingbot/connector/exchange/probit/probit_exchange.py @@ -367,7 +367,7 @@ async def _api_request(self, if method == "GET": response = await client.get(path_url, headers=headers, params=params) elif method == "POST": - response = await client.post(path_url, headers=headers, params=params, data=data) + response = await client.post(path_url, headers=headers, data=ujson.dumps(data)) else: raise NotImplementedError(f"{method} HTTP Method not implemented. ") @@ -477,8 +477,8 @@ async def _create_order(self, "type": "limit", # ProBit Order Types ["limit", "market"} "side": trade_type.name.lower(), # ProBit Order Sides ["buy", "sell"] "time_in_force": "gtc", # gtc = Good-Til-Cancelled - "limit_price": price, - "quantity": amount, + "limit_price": str(price), + "quantity": str(amount), "client_order_id": order_id } From e921befe7bc67c39034d7c64101239dcc505b615 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Thu, 18 Feb 2021 21:15:38 +0800 Subject: [PATCH 31/73] (add) add probit api key templates --- hummingbot/templates/conf_global_TEMPLATE.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hummingbot/templates/conf_global_TEMPLATE.yml b/hummingbot/templates/conf_global_TEMPLATE.yml index 8f77ed6df7..0f41f4d3b0 100644 --- a/hummingbot/templates/conf_global_TEMPLATE.yml +++ b/hummingbot/templates/conf_global_TEMPLATE.yml @@ -77,6 +77,9 @@ terra_wallet_seeds: null balancer_max_swaps: 4 +probit_api_key: null +probit_secret_key: null + # Ethereum wallet address: required for trading on a DEX ethereum_wallet: null ethereum_rpc_url: null From 28a3ad81ce912354a59ed7dee76d73bda00244aa Mon Sep 17 00:00:00 2001 From: Nullably <37262506+Nullably@users.noreply.github.com> Date: Thu, 18 Feb 2021 22:24:10 +0800 Subject: [PATCH 32/73] (feat) add parrot connector --- hummingbot/connector/parrot.py | 83 +++++++++++++++++++ .../liquidity_mining/liquidity_mining.py | 37 +++++++-- test/connector/test_parrot.py | 29 +++++++ 3 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 hummingbot/connector/parrot.py create mode 100644 test/connector/test_parrot.py diff --git a/hummingbot/connector/parrot.py b/hummingbot/connector/parrot.py new file mode 100644 index 0000000000..e7ada091ef --- /dev/null +++ b/hummingbot/connector/parrot.py @@ -0,0 +1,83 @@ +import aiohttp +from typing import List, Dict +from dataclasses import dataclass +from decimal import Decimal +from hummingbot.connector.exchange.binance.binance_utils import convert_from_exchange_trading_pair +from hummingbot.core.utils.async_utils import safe_gather + +PARROT_MINER_BASE_URL = "https://papi-development.hummingbot.io/v1/mining_data/" + +s_decimal_0 = Decimal("0") + + +@dataclass +class CampaignSummary: + market_id: int = 0 + trading_pair: str = "" + exchange_name: str = 0 + spread_max: Decimal = s_decimal_0 + payout_asset: str = "" + liquidity: Decimal = s_decimal_0 + active_bots: int = 0 + reward_per_day: Decimal = s_decimal_0 + apy: Decimal = s_decimal_0 + + +async def get_campaign_summary(exchange: str, trading_pairs: List[str] = []) -> Dict[str, CampaignSummary]: + campaigns = await get_active_campaigns(exchange, trading_pairs) + tasks = [get_market_snapshots(m_id) for m_id in campaigns] + results = await safe_gather(*tasks, return_exceptions=True) + for result in results: + if isinstance(result, Exception): + raise result + if result["items"]: + snapshot = result["items"][0] + market_id = int(snapshot["market_id"]) + campaign = campaigns[market_id] + campaign.apy = Decimal(snapshot["annualized_return"]) / Decimal("100") + reward = snapshot["payout_summary"]["open_volume"]["reward"] + if campaign.payout_asset in reward["ask"]: + campaign.reward_per_day = Decimal(str(reward["ask"][campaign.payout_asset])) + if campaign.payout_asset in reward["bid"]: + campaign.reward_per_day += Decimal(str(reward["bid"][campaign.payout_asset])) + oov = snapshot["summary_stats"]["open_volume"] + campaign.liquidity = Decimal(oov["oov_ask"]) + Decimal(oov["oov_bid"]) + campaign.active_bots = int(oov["bots"]) + return {c.trading_pair: c for c in campaigns.values()} + + +async def get_market_snapshots(market_id: int): + async with aiohttp.ClientSession() as client: + url = f"{PARROT_MINER_BASE_URL}market_snapshots/{market_id}?aggregate=1d" + resp = await client.get(url) + resp_json = await resp.json() + return resp_json + + +async def get_active_campaigns(exchange: str, trading_pairs: List[str] = []) -> Dict[int, CampaignSummary]: + campaigns = {} + async with aiohttp.ClientSession() as client: + url = f"{PARROT_MINER_BASE_URL}campaigns" + resp = await client.get(url) + resp_json = await resp.json() + for campaign_retval in resp_json: + for market in campaign_retval["markets"]: + if market["exchange_name"] != exchange: + continue + t_pair = market["trading_pair"] + if exchange == "binance": + t_pair = convert_from_exchange_trading_pair(t_pair) + if trading_pairs and t_pair not in trading_pairs: + continue + campaign = CampaignSummary() + campaign.market_id = int(market["id"]) + campaign.trading_pair = t_pair + campaign.exchange_name = market["exchange_name"] + campaigns[campaign.market_id] = campaign + for bounty_period in campaign_retval["bounty_periods"]: + for payout_parameter in bounty_period["payout_parameters"]: + market_id = int(payout_parameter["market_id"]) + if market_id in campaigns: + campaigns[market_id].spread_max = Decimal(str(payout_parameter["spread_max"])) / Decimal("100") + campaigns[market_id].payout_asset = payout_parameter["payout_asset"] + return campaigns diff --git a/hummingbot/strategy/liquidity_mining/liquidity_mining.py b/hummingbot/strategy/liquidity_mining/liquidity_mining.py index 8045de06b2..fd511f2721 100644 --- a/hummingbot/strategy/liquidity_mining/liquidity_mining.py +++ b/hummingbot/strategy/liquidity_mining/liquidity_mining.py @@ -15,9 +15,11 @@ from hummingbot.core.event.events import OrderType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.utils.estimate_fee import estimate_fee +from hummingbot.core.utils.market_price import usd_value from hummingbot.strategy.pure_market_making.inventory_skew_calculator import ( calculate_bid_ask_ratios_from_base_asset_ratio ) +from hummingbot.connector.parrot import get_campaign_summary NaN = float("nan") s_decimal_zero = Decimal(0) s_decimal_nan = Decimal("NaN") @@ -137,21 +139,42 @@ async def active_orders_df(self) -> pd.DataFrame: def market_status_df(self) -> pd.DataFrame: data = [] - columns = ["Exchange", "Market", "Mid Price", "Volatility", "Base Bal", "Quote Bal", " Base %"] + columns = ["Market", "Mid Price", "Volatility", "Base Bal", "Quote Bal", f"Budget ({self._token})", + "Base %", "Quote %"] for market, market_info in self._market_infos.items(): mid_price = market_info.get_mid_price() base_bal = self._sell_budgets[market] quote_bal = self._buy_budgets[market] total_bal = (base_bal * mid_price) + quote_bal + total_bal_in_token = total_bal + if not self.is_token_a_quote_token(): + total_bal_in_token = base_bal + (quote_bal / mid_price) base_pct = (base_bal * mid_price) / total_bal if total_bal > 0 else s_decimal_zero + quote_pct = quote_bal / total_bal if total_bal > 0 else s_decimal_zero data.append([ - self._exchange.display_name, market, float(mid_price), "" if self._volatility[market].is_nan() else f"{self._volatility[market]:.2%}", float(base_bal), float(quote_bal), - f"{base_pct:.0%}" + float(total_bal_in_token), + f"{base_pct:.0%}", + f"{quote_pct:.0%}" + ]) + return pd.DataFrame(data=data, columns=columns).replace(np.nan, '', regex=True) + + async def miner_status_df(self) -> pd.DataFrame: + data = [] + columns = ["Market", "Reward/day", "Liquidity (bots)", "Yield/day", "Max spread"] + campaigns = await get_campaign_summary(self._exchange.display_name, list(self._market_infos.keys())) + for market, campaign in campaigns.items(): + liquidity_usd = await usd_value(market.split('-')[0], campaign.liquidity) + data.append([ + market, + f"{campaign.reward_per_day:.2f} {campaign.payout_asset}", + f"${liquidity_usd:.0f} ({campaign.active_bots})", + f"{campaign.apy / Decimal(365):.2%}", + f"{campaign.spread_max:.2%}%" ]) return pd.DataFrame(data=data, columns=columns).replace(np.nan, '', regex=True) @@ -162,8 +185,12 @@ async def format_status(self) -> str: warning_lines = [] warning_lines.extend(self.network_warning(list(self._market_infos.values()))) - lines.extend(["", " Markets:"] + [" " + line for line in - self.market_status_df().to_string(index=False).split("\n")]) + market_df = self.market_status_df() + lines.extend(["", " Markets:"] + [" " + line for line in market_df.to_string(index=False).split("\n")]) + + miner_df = await self.miner_status_df() + if not miner_df.empty: + lines.extend(["", " Miners:"] + [" " + line for line in miner_df.to_string(index=False).split("\n")]) # See if there're any open orders. if len(self.active_orders) > 0: diff --git a/test/connector/test_parrot.py b/test/connector/test_parrot.py new file mode 100644 index 0000000000..0c8491b8c8 --- /dev/null +++ b/test/connector/test_parrot.py @@ -0,0 +1,29 @@ +from os.path import join, realpath +import sys; sys.path.insert(0, realpath(join(__file__, "../../../../"))) +import unittest +import asyncio +from hummingbot.connector.parrot import get_active_campaigns, get_campaign_summary + + +class ParrotConnectorUnitTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + + def test_get_active_campaigns(self): + self.ev_loop.run_until_complete(self._test_get_active_campaigns()) + + async def _test_get_active_campaigns(self): + results = await get_active_campaigns("binance") + self.assertGreater(len(results), 0) + for result in results.values(): + print(result) + + def test_get_campaign_summary(self): + self.ev_loop.run_until_complete(self._test_get_campaign_summary()) + + async def _test_get_campaign_summary(self): + results = await get_campaign_summary("binance", ["RLC-BTC", "RLC-ETH"]) + self.assertLessEqual(len(results), 2) + for result in results.values(): + print(result) From 1c932e86de3324abdf80824496d74e710f61ba83 Mon Sep 17 00:00:00 2001 From: vic-en Date: Thu, 18 Feb 2021 19:54:01 +0100 Subject: [PATCH 33/73] (feat) refactor fetch_trading_pairs to *_api_order_book.py file --- ...tual_finance_api_order_book_data_source.py | 43 +++++++++++++++++++ .../perpetual_finance_derivative.py | 35 +-------------- 2 files changed, 44 insertions(+), 34 deletions(-) create mode 100644 hummingbot/connector/derivative/perpetual_finance/perpetual_finance_api_order_book_data_source.py diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_api_order_book_data_source.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_api_order_book_data_source.py new file mode 100644 index 0000000000..4abc57b74c --- /dev/null +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_api_order_book_data_source.py @@ -0,0 +1,43 @@ +import aiohttp +from typing import List +import json +import ssl + +from hummingbot.client.config.global_config_map import global_config_map +from hummingbot.connector.derivative.perpetual_finance.perpetual_finance_utils import convert_from_exchange_trading_pair +from hummingbot.client.settings import GATEAWAY_CA_CERT_PATH, GATEAWAY_CLIENT_CERT_PATH, GATEAWAY_CLIENT_KEY_PATH + + +class PerpetualFinanceAPIOrderBookDataSource: + @staticmethod + async def fetch_trading_pairs() -> List[str]: + ssl_ctx = ssl.create_default_context(cafile=GATEAWAY_CA_CERT_PATH) + ssl_ctx.load_cert_chain(GATEAWAY_CLIENT_CERT_PATH, GATEAWAY_CLIENT_KEY_PATH) + conn = aiohttp.TCPConnector(ssl_context=ssl_ctx) + client = aiohttp.ClientSession(connector=conn) + + base_url = f"https://{global_config_map['gateway_api_host'].value}:" \ + f"{global_config_map['gateway_api_port'].value}/perpfi/" + response = await client.get(base_url + "pairs") + parsed_response = json.loads(await response.text()) + if response.status != 200: + err_msg = "" + if "error" in parsed_response: + err_msg = f" Message: {parsed_response['error']}" + raise IOError(f"Error fetching pairs from gateway. HTTP status is {response.status}.{err_msg}") + pairs = parsed_response.get("pairs", []) + if "error" in parsed_response or len(pairs) == 0: + raise Exception(f"Error: {parsed_response['error']}") + else: + status = await client.get(base_url) + status = json.loads(await status.text()) + loadedMetadata = status["loadedMetadata"] + while (not loadedMetadata): + resp = await client.get(base_url + "load-metadata") + resp = json.loads(await resp.text()) + loadedMetadata = resp.get("loadedMetadata", False) + return PerpetualFinanceAPIOrderBookDataSource.fetch_trading_pairs() + trading_pairs = [] + for pair in pairs: + trading_pairs.append(convert_from_exchange_trading_pair(pair)) + return trading_pairs diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py index 71bb2ffad7..9323d76967 100644 --- a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py @@ -32,7 +32,7 @@ ) from hummingbot.connector.derivative_base import DerivativeBase from hummingbot.connector.derivative.perpetual_finance.perpetual_finance_in_flight_order import PerpetualFinanceInFlightOrder -from hummingbot.connector.derivative.perpetual_finance.perpetual_finance_utils import convert_to_exchange_trading_pair, convert_from_exchange_trading_pair +from hummingbot.connector.derivative.perpetual_finance.perpetual_finance_utils import convert_to_exchange_trading_pair from hummingbot.client.settings import GATEAWAY_CA_CERT_PATH, GATEAWAY_CLIENT_CERT_PATH, GATEAWAY_CLIENT_KEY_PATH from hummingbot.client.config.global_config_map import global_config_map from hummingbot.connector.derivative.position import Position @@ -92,39 +92,6 @@ def __init__(self, def name(self): return "perpetual_finance" - @staticmethod - async def fetch_trading_pairs() -> List[str]: - ssl_ctx = ssl.create_default_context(cafile=GATEAWAY_CA_CERT_PATH) - ssl_ctx.load_cert_chain(GATEAWAY_CLIENT_CERT_PATH, GATEAWAY_CLIENT_KEY_PATH) - conn = aiohttp.TCPConnector(ssl_context=ssl_ctx) - client = aiohttp.ClientSession(connector=conn) - - base_url = f"https://{global_config_map['gateway_api_host'].value}:" \ - f"{global_config_map['gateway_api_port'].value}/perpfi/" - response = await client.get(base_url + "pairs") - parsed_response = json.loads(await response.text()) - if response.status != 200: - err_msg = "" - if "error" in parsed_response: - err_msg = f" Message: {parsed_response['error']}" - raise IOError(f"Error fetching pairs from gateway. HTTP status is {response.status}.{err_msg}") - pairs = parsed_response.get("pairs", []) - if "error" in parsed_response or len(pairs) == 0: - raise Exception(f"Error: {parsed_response['error']}") - else: - status = await client.get(base_url) - status = json.loads(await status.text()) - loadedMetadata = status["loadedMetadata"] - while (not loadedMetadata): - resp = await client.get(base_url + "load-metadata") - resp = json.loads(await resp.text()) - loadedMetadata = resp.get("loadedMetadata", False) - return PerpetualFinanceDerivative.fetch_trading_pairs() - trading_pairs = [] - for pair in pairs: - trading_pairs.append(convert_from_exchange_trading_pair(pair)) - return trading_pairs - @property def limit_orders(self) -> List[LimitOrder]: return [ From 57b48217eb37f99b08cb53c02ee936a4e25c9872 Mon Sep 17 00:00:00 2001 From: vic-en Date: Fri, 19 Feb 2021 12:24:41 +0100 Subject: [PATCH 34/73] (feat) refactor autocomplete filter for connector name selection for all strategies --- hummingbot/client/ui/completer.py | 49 ++++++++++++------- .../strategy/amm_arb/amm_arb_config_map.py | 4 +- .../arbitrage/arbitrage_config_map.py | 4 +- .../strategy/celo_arb/celo_arb_config_map.py | 2 +- ...cross_exchange_market_making_config_map.py | 4 +- .../liquidity_mining_config_map.py | 2 +- .../perpetual_market_making_config_map.py | 2 +- .../pure_market_making_config_map.py | 2 +- 8 files changed, 40 insertions(+), 29 deletions(-) diff --git a/hummingbot/client/ui/completer.py b/hummingbot/client/ui/completer.py index 527c442602..ef85af2ccb 100644 --- a/hummingbot/client/ui/completer.py +++ b/hummingbot/client/ui/completer.py @@ -14,7 +14,8 @@ DERIVATIVES, STRATEGIES, CONF_FILE_PATH, - SCRIPTS_PATH + SCRIPTS_PATH, + ConnectorType ) from hummingbot.client.ui.parser import ThrowingArgumentParser from hummingbot.core.utils.wallet_setup import list_wallets @@ -28,15 +29,21 @@ def file_name_list(path, file_extension): return sorted([f for f in listdir(path) if isfile(join(path, f)) and f.endswith(file_extension)]) +SPOT_PROTOCOL_CONNECTOR = {x.name for x in CONNECTOR_SETTINGS.values() if x.type == ConnectorType.Connector} +DERIVATIVE_PROTOCOL_CONNECTOR = {x.name for x in CONNECTOR_SETTINGS.values() if x.type == ConnectorType.Derivative and not x.centralised} + + class HummingbotCompleter(Completer): def __init__(self, hummingbot_application): super(HummingbotCompleter, self).__init__() self.hummingbot_application = hummingbot_application self._path_completer = WordCompleter(file_name_list(CONF_FILE_PATH, "yml")) self._command_completer = WordCompleter(self.parser.commands, ignore_case=True) - self._connector_completer = WordCompleter(CONNECTOR_SETTINGS.keys(), ignore_case=True) - self._exchange_completer = WordCompleter(EXCHANGES, ignore_case=True) + self._exchange_completer = WordCompleter(CONNECTOR_SETTINGS.keys(), ignore_case=True) + self._spot_completer = WordCompleter(EXCHANGES.union(SPOT_PROTOCOL_CONNECTOR), ignore_case=True) + self._spot_exchange_completer = WordCompleter(EXCHANGES, ignore_case=True) self._derivative_completer = WordCompleter(DERIVATIVES, ignore_case=True) + self._derivative_exchange_completer = WordCompleter(DERIVATIVES.difference(DERIVATIVE_PROTOCOL_CONNECTOR), ignore_case=True) self._connect_option_completer = WordCompleter(CONNECT_OPTIONS, ignore_case=True) self._export_completer = WordCompleter(["keys", "trades"], ignore_case=True) self._balance_completer = WordCompleter(["limit", "paper"], ignore_case=True) @@ -96,25 +103,21 @@ def _complete_options(self, document: Document) -> bool: return "(" in self.prompt_text and ")" in self.prompt_text and "/" in self.prompt_text def _complete_exchanges(self, document: Document) -> bool: - text_before_cursor: str = document.text_before_cursor - return "-e" in text_before_cursor or \ - "--exchange" in text_before_cursor or \ - any(x for x in ("exchange name", "name of exchange", "name of the exchange") + return any(x for x in ("exchange name", "name of exchange", "name of the exchange") if x in self.prompt_text.lower()) def _complete_derivatives(self, document: Document) -> bool: text_before_cursor: str = document.text_before_cursor - return "--exchange" in text_before_cursor or \ - "perpetual" in text_before_cursor or \ - any(x for x in ("derivative name", "name of derivative", "name of the derivative") + return "perpetual" in text_before_cursor or \ + any(x for x in ("derivative connector", "derivative name", "name of derivative", "name of the derivative") if x in self.prompt_text.lower()) def _complete_connect_options(self, document: Document) -> bool: text_before_cursor: str = document.text_before_cursor return text_before_cursor.startswith("connect ") - def _complete_connectors(self, document: Document) -> bool: - return "connector" in self.prompt_text + def _complete_spot_connectors(self, document: Document) -> bool: + return "spot" in self.prompt_text def _complete_export_options(self, document: Document) -> bool: text_before_cursor: str = document.text_before_cursor @@ -175,9 +178,13 @@ def get_completions(self, document: Document, complete_event: CompleteEvent): for c in self._wallet_address_completer.get_completions(document, complete_event): yield c - elif self._complete_connectors(document): - for c in self._connector_completer.get_completions(document, complete_event): - yield c + elif self._complete_spot_connectors(document): + if "(Exchange/AMM)" in self.prompt_text: + for c in self._spot_completer.get_completions(document, complete_event): + yield c + else: + for c in self._spot_exchange_completer.get_completions(document, complete_event): + yield c elif self._complete_connect_options(document): for c in self._connect_option_completer.get_completions(document, complete_event): @@ -199,14 +206,18 @@ def get_completions(self, document: Document, complete_event: CompleteEvent): for c in self._history_completer.get_completions(document, complete_event): yield c + elif self._complete_derivatives(document): + if "(Exchange/AMM)" in self.prompt_text: + for c in self._derivative_completer.get_completions(document, complete_event): + yield c + else: + for c in self._derivative_exchange_completer.get_completions(document, complete_event): + yield c + elif self._complete_exchanges(document): for c in self._exchange_completer.get_completions(document, complete_event): yield c - elif self._complete_derivatives(document): - for c in self._derivative_completer.get_completions(document, complete_event): - yield c - elif self._complete_trading_pairs(document): for c in self._trading_pair_completer.get_completions(document, complete_event): yield c diff --git a/hummingbot/strategy/amm_arb/amm_arb_config_map.py b/hummingbot/strategy/amm_arb/amm_arb_config_map.py index 16e15e90c7..d85d87951c 100644 --- a/hummingbot/strategy/amm_arb/amm_arb_config_map.py +++ b/hummingbot/strategy/amm_arb/amm_arb_config_map.py @@ -51,7 +51,7 @@ def order_amount_prompt() -> str: default="amm_arb"), "connector_1": ConfigVar( key="connector_1", - prompt="Enter your first connector (exchange/AMM) >>> ", + prompt="Enter your first spot connector (Exchange/AMM) >>> ", prompt_on_new=True, validator=validate_connector, on_validated=exchange_on_validated), @@ -62,7 +62,7 @@ def order_amount_prompt() -> str: on_validated=market_1_on_validated), "connector_2": ConfigVar( key="connector_2", - prompt="Enter your second connector (exchange/AMM) >>> ", + prompt="Enter your second spot connector (Exchange/AMM) >>> ", prompt_on_new=True, validator=validate_connector, on_validated=exchange_on_validated), diff --git a/hummingbot/strategy/arbitrage/arbitrage_config_map.py b/hummingbot/strategy/arbitrage/arbitrage_config_map.py index 278f71cdde..96313098ae 100644 --- a/hummingbot/strategy/arbitrage/arbitrage_config_map.py +++ b/hummingbot/strategy/arbitrage/arbitrage_config_map.py @@ -47,13 +47,13 @@ def secondary_market_on_validated(value: str): default="arbitrage"), "primary_market": ConfigVar( key="primary_market", - prompt="Enter your primary exchange name >>> ", + prompt="Enter your primary spot connector >>> ", prompt_on_new=True, validator=validate_exchange, on_validated=lambda value: required_exchanges.append(value)), "secondary_market": ConfigVar( key="secondary_market", - prompt="Enter your secondary exchange name >>> ", + prompt="Enter your secondary spot connector >>> ", prompt_on_new=True, validator=validate_exchange, on_validated=secondary_market_on_validated), diff --git a/hummingbot/strategy/celo_arb/celo_arb_config_map.py b/hummingbot/strategy/celo_arb/celo_arb_config_map.py index c92f5b56b0..3e99ca39b3 100644 --- a/hummingbot/strategy/celo_arb/celo_arb_config_map.py +++ b/hummingbot/strategy/celo_arb/celo_arb_config_map.py @@ -35,7 +35,7 @@ def order_amount_prompt() -> str: default="celo_arb"), "secondary_exchange": ConfigVar( key="secondary_exchange", - prompt="Enter your secondary exchange name >>> ", + prompt="Enter your secondary spot connector >>> ", prompt_on_new=True, validator=validate_exchange, on_validated=exchange_on_validated), diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py index a87eb8f8f6..2d8f2438a9 100644 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py @@ -78,14 +78,14 @@ def taker_market_on_validated(value: str): ), "maker_market": ConfigVar( key="maker_market", - prompt="Enter your maker exchange name >>> ", + prompt="Enter your maker spot connector >>> ", prompt_on_new=True, validator=validate_exchange, on_validated=lambda value: required_exchanges.append(value), ), "taker_market": ConfigVar( key="taker_market", - prompt="Enter your taker exchange name >>> ", + prompt="Enter your taker spot connector >>> ", prompt_on_new=True, validator=validate_exchange, on_validated=taker_market_on_validated, diff --git a/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py b/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py index 891733234f..ff421ead7b 100644 --- a/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py +++ b/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py @@ -37,7 +37,7 @@ def order_size_prompt() -> str: default="liquidity_mining"), "exchange": ConfigVar(key="exchange", - prompt="Enter your liquidity mining exchange name >>> ", + prompt="Enter the spot connector to use for liquidity mining >>> ", validator=validate_exchange, on_validated=exchange_on_validated, prompt_on_new=True), diff --git a/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py b/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py index b18277dc9d..1040ef9c53 100644 --- a/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py +++ b/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py @@ -126,7 +126,7 @@ def derivative_on_validated(value: str): default="perpetual_market_making"), "derivative": ConfigVar(key="derivative", - prompt="Enter your maker derivative name >>> ", + prompt="Enter your maker derivative connector >>> ", validator=validate_derivative, on_validated=derivative_on_validated, prompt_on_new=True), diff --git a/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py b/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py index a8d69eec0b..7f8065b698 100644 --- a/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py +++ b/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py @@ -111,7 +111,7 @@ def exchange_on_validated(value: str): default="pure_market_making"), "exchange": ConfigVar(key="exchange", - prompt="Enter your maker exchange name >>> ", + prompt="Enter your maker spot connector >>> ", validator=validate_exchange, on_validated=exchange_on_validated, prompt_on_new=True), From 5229dc159cd75a771cdeec3d695b2a5115ca9f42 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Sun, 21 Feb 2021 02:25:10 +0800 Subject: [PATCH 35/73] (fix) trades not being process adequetely due to Decimal conversion bug --- hummingbot/connector/exchange/probit/probit_exchange.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hummingbot/connector/exchange/probit/probit_exchange.py b/hummingbot/connector/exchange/probit/probit_exchange.py index c05d6cf996..30fc2d6038 100644 --- a/hummingbot/connector/exchange/probit/probit_exchange.py +++ b/hummingbot/connector/exchange/probit/probit_exchange.py @@ -674,6 +674,10 @@ async def _update_order_status(self): min_ts: float = float("inf") for order_update in order_results: + if isinstance(order_update, Exception): + raise order_update + + # Order Creation Time order_ts: float = probit_utils.convert_iso_to_epoch(order_update["data"]["time"]) if order_ts < min_ts: @@ -752,6 +756,10 @@ def _process_trade_message(self, order_msg: Dict[str, Any]): Updates in-flight order and trigger order filled event for trade message received. Triggers order completed event if the total executed amount equals to the specified order amount. """ + # Only process trade when trade fees have been accounted for; when trade status is "settled". + if order_msg["status"] != "settled": + return + ex_order_id = order_msg["order_id"] client_order_id = None From 3cb6cca36d1f04f0c7aa1b0e5a430675313020dc Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Sun, 21 Feb 2021 18:41:08 +0800 Subject: [PATCH 36/73] (fix) INVALID_JSON error in _update_order_status --- .../exchange/probit/probit_exchange.py | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/hummingbot/connector/exchange/probit/probit_exchange.py b/hummingbot/connector/exchange/probit/probit_exchange.py index 30fc2d6038..a78d3d3f76 100644 --- a/hummingbot/connector/exchange/probit/probit_exchange.py +++ b/hummingbot/connector/exchange/probit/probit_exchange.py @@ -660,7 +660,7 @@ async def _update_order_status(self): "order_id": ex_order_id } - tasks.append(self._api_request(method="POST", + tasks.append(self._api_request(method="GET", path_url=CONSTANTS.ORDER_URL, params=query_params, is_auth_required=True) @@ -678,11 +678,11 @@ async def _update_order_status(self): raise order_update # Order Creation Time - order_ts: float = probit_utils.convert_iso_to_epoch(order_update["data"]["time"]) - - if order_ts < min_ts: - min_order_ts = order_update["data"]["time"] - min_ts = order_ts + for update in order_update["data"]: + order_ts: float = probit_utils.convert_iso_to_epoch(update["time"]) + if order_ts < min_ts: + min_order_ts = update["time"] + min_ts = order_ts trade_history_tasks = [] for trading_pair in self._trading_pairs: @@ -694,7 +694,9 @@ async def _update_order_status(self): } trade_history_tasks.append(self._api_request( method="GET", - path_url=CONSTANTS.TRADE_HISTORY_URL + path_url=CONSTANTS.TRADE_HISTORY_URL, + params=query_params, + is_auth_required=True )) trade_history_results: List[Dict[str, Any]] = await safe_gather(*trade_history_tasks, return_exceptions=True) @@ -709,16 +711,15 @@ async def _update_order_status(self): for trade in trade_details: self._process_trade_message(trade) - for update_result in order_results: - if isinstance(update_result, Exception): - raise update_result - if "data" not in update_result: - self.logger().info(f"_update_order_status data not in resp: {update_result}") + for order_update in order_results: + if isinstance(order_update, Exception): + raise order_update + if "data" not in order_update: + self.logger().info(f"_update_order_status data not in resp: {order_update}") continue - order_details: List[Dict[str, Any]] = update_result["data"] - for order in order_details: - self._process_order_message(order_details) + for order in order_update["data"]: + self._process_order_message(order) def _process_order_message(self, order_msg: Dict[str, Any]): """ From 747df25340b065b032ecb2b64c5c94634de8b4d5 Mon Sep 17 00:00:00 2001 From: vic-en Date: Mon, 22 Feb 2021 13:28:32 +0100 Subject: [PATCH 37/73] (feat) hardcode perpetual finance along other gateway connectors --- hummingbot/client/hummingbot_application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/hummingbot_application.py b/hummingbot/client/hummingbot_application.py index c64d0b3ccb..3566917232 100644 --- a/hummingbot/client/hummingbot_application.py +++ b/hummingbot/client/hummingbot_application.py @@ -241,7 +241,7 @@ def _initialize_markets(self, market_names: List[Tuple[str, List[str]]]): if conn_setting.use_ethereum_wallet: ethereum_rpc_url = global_config_map.get("ethereum_rpc_url").value # Todo: Hard coded this execption for now until we figure out how to handle all ethereum connectors. - if connector_name in ["balancer", "uniswap"]: + if connector_name in ["balancer", "uniswap", "perpetual_finance"]: private_key = get_eth_wallet_private_key() init_params.update(wallet_private_key=private_key, ethereum_rpc_url=ethereum_rpc_url) else: From c3b895b1b71bf8ed349d1d29722be296c0da1b39 Mon Sep 17 00:00:00 2001 From: vic-en Date: Mon, 22 Feb 2021 13:30:57 +0100 Subject: [PATCH 38/73] (feat) add XDAI to balance command --- hummingbot/client/command/balance_command.py | 20 ++++++++++++++++++-- hummingbot/strategy/strategy_base.pyx | 3 ++- hummingbot/user/user_balances.py | 10 ++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/hummingbot/client/command/balance_command.py b/hummingbot/client/command/balance_command.py index cd4a1c0508..171485ee0a 100644 --- a/hummingbot/client/command/balance_command.py +++ b/hummingbot/client/command/balance_command.py @@ -110,11 +110,17 @@ async def show_balances(self): eth_address = global_config_map["ethereum_wallet"].value if eth_address is not None: - df = await self.ethereum_balances_df() - lines = [" " + line for line in df.to_string(index=False).split("\n")] + eth_df = await self.ethereum_balances_df() + lines = [" " + line for line in eth_df.to_string(index=False).split("\n")] self._notify("\nethereum:") self._notify("\n".join(lines)) + # XDAI balances + xdai_df = await self.xdai_balances_df() + lines = [" " + line for line in xdai_df.to_string(index=False).split("\n")] + self._notify("\nxdai:") + self._notify("\n".join(lines)) + async def exchange_balances_df(self, # type: HummingbotApplication exchange_balances: Dict[str, Decimal], exchange_limits: Dict[str, str]): @@ -177,6 +183,16 @@ async def ethereum_balances_df(self, # type: HummingbotApplication df.sort_values(by=["Asset"], inplace=True) return df + async def xdai_balances_df(self, # type: HummingbotApplication + ): + rows = [] + bals = await UserBalances.xdai_balances() + for token, bal in bals.items(): + rows.append({"Asset": token, "Amount": round(bal, 4)}) + df = pd.DataFrame(data=rows, columns=["Asset", "Amount"]) + df.sort_values(by=["Asset"], inplace=True) + return df + async def asset_limits_df(self, asset_limit_conf: Dict[str, str]): rows = [] diff --git a/hummingbot/strategy/strategy_base.pyx b/hummingbot/strategy/strategy_base.pyx index d214b288e8..610cb5eb80 100755 --- a/hummingbot/strategy/strategy_base.pyx +++ b/hummingbot/strategy/strategy_base.pyx @@ -19,6 +19,7 @@ from hummingbot.core.event.events import ( ) from .order_tracker import OrderTracker +from hummingbot.connector.derivative_base import DerivativeBase NaN = float("nan") s_decimal_nan = Decimal("NaN") @@ -213,7 +214,7 @@ cdef class StrategyBase(TimeIterator): for market_trading_pair_tuple in market_trading_pair_tuples: base_balance = market_trading_pair_tuple.market.get_balance(market_trading_pair_tuple.base_asset) quote_balance = market_trading_pair_tuple.market.get_balance(market_trading_pair_tuple.quote_asset) - if base_balance <= Decimal("0.0001"): + if base_balance <= Decimal("0.0001") and not isinstance(market_trading_pair_tuple.market, DerivativeBase): warning_lines.append(f" {market_trading_pair_tuple.market.name} market " f"{market_trading_pair_tuple.base_asset} balance is too low. Cannot place order.") if quote_balance <= Decimal("0.0001"): diff --git a/hummingbot/user/user_balances.py b/hummingbot/user/user_balances.py index 326a80549e..b9e6e313c2 100644 --- a/hummingbot/user/user_balances.py +++ b/hummingbot/user/user_balances.py @@ -5,6 +5,7 @@ from hummingbot.core.utils.async_utils import safe_gather from hummingbot.client.config.global_config_map import global_config_map from hummingbot.connector.connector.balancer.balancer_connector import BalancerConnector +from hummingbot.connector.derivative.perpetual_finance.perpetual_finance_derivative import PerpetualFinanceDerivative from hummingbot.client.settings import ethereum_required_trading_pairs from typing import Optional, Dict, List from decimal import Decimal @@ -122,6 +123,15 @@ async def eth_n_erc20_balances() -> Dict[str, Decimal]: await connector._update_balances() return connector.get_all_balances() + @staticmethod + async def xdai_balances() -> Dict[str, Decimal]: + connector = PerpetualFinanceDerivative("", + get_eth_wallet_private_key(), + "", + True) + await connector._update_balances() + return connector.get_all_balances() + @staticmethod def validate_ethereum_wallet() -> Optional[str]: if global_config_map.get("ethereum_wallet").value is None: From 26c51ff2d66d208b452679e9aac7a100218e9f8a Mon Sep 17 00:00:00 2001 From: vic-en Date: Mon, 22 Feb 2021 13:32:47 +0100 Subject: [PATCH 39/73] (fix) fix errors related to funding_info in derivative connectors --- .../binance_perpetual_derivative.py | 47 ++++++++----------- .../perpetual_finance_derivative.py | 33 +++++++++---- hummingbot/connector/derivative_base.py | 2 +- hummingbot/model/funding_payment.py | 4 +- 4 files changed, 45 insertions(+), 41 deletions(-) diff --git a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py index f2d4359d35..8a612e7890 100644 --- a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py +++ b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py @@ -129,8 +129,7 @@ def __init__(self, self._order_not_found_records = {} self._last_timestamp = 0 self._trading_rules = {} - # self._trade_fees = {} - # self._last_update_trade_fees_timestamp = 0 + self._trading_pairs = trading_pairs self._status_polling_task = None self._user_stream_event_listener_task = None self._trading_rules_polling_task = None @@ -161,9 +160,7 @@ def status_dict(self): "order_books_initialized": self._order_book_tracker.ready, "account_balance": len(self._account_balances) > 0 if self._trading_required else True, "trading_rule_initialized": len(self._trading_rules) > 0, - - # TODO: Uncomment when figured out trade fees - # "trade_fees_initialized": len(self._trade_fees) > 0 + "funding_info": len(self._funding_info) > 0 } @property @@ -630,8 +627,8 @@ def get_order_book(self, trading_pair: str) -> OrderBook: return order_books[trading_pair] async def _update_trading_rules(self): - last_tick = self._last_timestamp / 60.0 - current_tick = self.current_timestamp / 60.0 + last_tick = int(self._last_timestamp / 60.0) + current_tick = int(self.current_timestamp / 60.0) if current_tick > last_tick or len(self._trading_rules) < 1: exchange_info = await self.request(path="/fapi/v1/exchangeInfo", method=MethodType.GET, is_signed=False) trading_rules_list = self._format_trading_rules(exchange_info) @@ -674,11 +671,8 @@ async def _trading_rules_polling_loop(self): try: await safe_gather( self._update_trading_rules() - - # TODO: Uncomment when implemented - # self._update_trade_fees() ) - await asyncio.sleep(60) + await asyncio.sleep(3600) except asyncio.CancelledError: raise except Exception: @@ -692,25 +686,22 @@ async def _funding_info_polling_loop(self): try: ws_subscription_path: str = "/".join([f"{convert_to_exchange_trading_pair(trading_pair).lower()}@markPrice" for trading_pair in self._trading_pairs]) - stream_url: str = f"{self._stream_url}?streams={ws_subscription_path}" + stream_url: str = f"{self._stream_url}/stream?streams={ws_subscription_path}" async with websockets.connect(stream_url) as ws: ws: websockets.WebSocketClientProtocol = ws - try: - while True: - try: - raw_msg: str = await asyncio.wait_for(ws.recv(), timeout=10.0) - msg = ujson.loads(raw_msg) - trading_pair = msg["s"] - self._funding_info[trading_pair] = {"indexPrice": msg["i"], - "markPrice": msg["p"], - "nextFundingTime": msg["T"], - "rate": msg["r"]} - except asyncio.TimeoutError: - await ws.pong(data=b'') - except ConnectionClosed: - continue - finally: - await ws.close() + while True: + try: + raw_msg: str = await asyncio.wait_for(ws.recv(), timeout=10.0) + msg = ujson.loads(raw_msg) + trading_pair = convert_from_exchange_trading_pair(msg["data"]["s"]) + self._funding_info[trading_pair] = {"indexPrice": msg["data"]["i"], + "markPrice": msg["data"]["p"], + "nextFundingTime": msg["data"]["T"], + "rate": msg["data"]["r"]} + except asyncio.TimeoutError: + await ws.pong(data=b'') + except ConnectionClosed: + raise except asyncio.CancelledError: raise except Exception: diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py index 9323d76967..48b86ea17f 100644 --- a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py @@ -63,13 +63,12 @@ def logger(cls) -> HummingbotLogger: def __init__(self, trading_pairs: List[str], wallet_private_key: str, - ethereum_rpc_url: str, + ethereum_rpc_url: str, # not used, but left in place to be consistent with other gateway connectors trading_required: bool = True ): """ :param trading_pairs: a list of trading pairs :param wallet_private_key: a private key for eth wallet - :param ethereum_rpc_url: this is usually infura RPC URL :param trading_required: Whether actual trading is needed. """ super().__init__() @@ -407,13 +406,15 @@ def has_allowances(self) -> bool: def status_dict(self) -> Dict[str, bool]: return { "account_balance": len(self._account_balances) > 0 if self._trading_required else True, - "allowances": self.has_allowances() if self._trading_required else True + "allowances": self.has_allowances() if self._trading_required else True, + "funding_info": len(self._funding_info) > 0 } async def start_network(self): if self._trading_required: self._status_polling_task = safe_ensure_future(self._status_polling_loop()) self._auto_approve_task = safe_ensure_future(self.auto_approve()) + self._funding_info_polling_task = safe_ensure_future(self._funding_info_polling_loop()) async def stop_network(self): if self._status_polling_task is not None: @@ -422,6 +423,9 @@ async def stop_network(self): if self._auto_approve_task is not None: self._auto_approve_task.cancel() self._auto_approve_task = None + if self._funding_info_polling_task is not None: + self._funding_info_polling_task.cancel() + self._funding_info_polling_task = None async def check_network(self) -> NetworkStatus: try: @@ -491,7 +495,6 @@ async def _update_balances(self): async def _update_positions(self): position_tasks = [] funding_payment_tasks = [] - funding_info_tasks = [] for pair in self._trading_pairs: position_tasks.append(self._api_request("post", "perpfi/position", @@ -499,12 +502,8 @@ async def _update_positions(self): funding_payment_tasks.append(self._api_request("get", "perpfi/funding_payment", {"pair": convert_to_exchange_trading_pair(pair)})) - funding_info_tasks.append(self._api_request("get", - "perpfi/funding", - {"pair": convert_to_exchange_trading_pair(pair)})) positions = await safe_gather(*position_tasks, return_exceptions=True) funding_payments = await safe_gather(*funding_payment_tasks, return_exceptions=True) - funding_infos = await safe_gather(*funding_info_tasks, return_exceptions=True) for trading_pair, position in zip(self._trading_pairs, positions): position = position.get("position", {}) position_side = PositionSide.LONG if position.get("size", 0) > 0 else PositionSide.SHORT @@ -537,8 +536,22 @@ async def _update_positions(self): symbol=trading_pair, amount=payment)) - for trading_pair, funding_info in zip(self._trading_pairs, funding_infos): - self._funding_info[trading_pair] = funding_info["fr"] + async def _funding_info_polling_loop(self): + while True: + try: + funding_info_tasks = [] + for pair in self._trading_pairs: + funding_info_tasks.append(self._api_request("post", + "perpfi/funding", + {"pair": convert_to_exchange_trading_pair(pair)})) + funding_infos = await safe_gather(*funding_info_tasks, return_exceptions=True) + for trading_pair, funding_info in zip(self._trading_pairs, funding_infos): + self._funding_info[trading_pair] = funding_info["fr"] + except Exception: + self.logger().network("Unexpected error while fetching funding info.", exc_info=True, + app_warning_msg="Could not fetch new funding info from Perpetual Finance protocol. " + "Check network connection on gateway.") + await asyncio.sleep(30) def get_funding_info(self, trading_pair): return self._funding_info[trading_pair] diff --git a/hummingbot/connector/derivative_base.py b/hummingbot/connector/derivative_base.py index 5bd2667e33..cd8800f073 100644 --- a/hummingbot/connector/derivative_base.py +++ b/hummingbot/connector/derivative_base.py @@ -18,7 +18,7 @@ def __init__(self): self._funding_info = {} self._account_positions = {} self._position_mode = None - self._leverage = 1 + self._leverage = {} self._funding_payment_span = [0, 0] # time span(in seconds) before and after funding period when exchanges consider active positions eligible for funding payment def set_position_mode(self, position_mode: PositionMode): diff --git a/hummingbot/model/funding_payment.py b/hummingbot/model/funding_payment.py index 99faa771a6..cd11bf1dec 100644 --- a/hummingbot/model/funding_payment.py +++ b/hummingbot/model/funding_payment.py @@ -21,9 +21,9 @@ class FundingPayment(HummingbotBase): __tablename__ = "FundingPayment" - __table_args__ = (Index("tf_config_timestamp_index", + __table_args__ = (Index("fp_config_timestamp_index", "config_file_path", "timestamp"), - Index("tf_market_trading_pair_timestamp_index", + Index("fp_market_trading_pair_timestamp_index", "market", "symbol", "timestamp") ) From 7e7a3e95350e63d8ce8270e1de43790398fd30a6 Mon Sep 17 00:00:00 2001 From: vic-en Date: Mon, 22 Feb 2021 13:52:48 +0100 Subject: [PATCH 40/73] (feat) add spot-perp arbitrage strategy --- .../spot_perpetual_arbitrage/__init__.py | 0 .../spot_perpetual_arbitrage/arb_proposal.py | 125 +++++ .../spot_perpetual_arbitrage/dummy.pxd | 2 + .../spot_perpetual_arbitrage/dummy.pyx | 2 + .../spot_perpetual_arbitrage.py | 497 ++++++++++++++++++ .../spot_perpetual_arbitrage_config_map.py | 125 +++++ .../spot_perpetual_arbitrage/start.py | 37 ++ .../spot_perpetual_arbitrage/utils.py | 44 ++ ..._perpetual_arbitrage_strategy_TEMPLATE.yml | 37 ++ 9 files changed, 869 insertions(+) create mode 100644 hummingbot/strategy/spot_perpetual_arbitrage/__init__.py create mode 100644 hummingbot/strategy/spot_perpetual_arbitrage/arb_proposal.py create mode 100644 hummingbot/strategy/spot_perpetual_arbitrage/dummy.pxd create mode 100644 hummingbot/strategy/spot_perpetual_arbitrage/dummy.pyx create mode 100644 hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py create mode 100644 hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py create mode 100644 hummingbot/strategy/spot_perpetual_arbitrage/start.py create mode 100644 hummingbot/strategy/spot_perpetual_arbitrage/utils.py create mode 100644 hummingbot/templates/conf_spot_perpetual_arbitrage_strategy_TEMPLATE.yml diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/__init__.py b/hummingbot/strategy/spot_perpetual_arbitrage/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/arb_proposal.py b/hummingbot/strategy/spot_perpetual_arbitrage/arb_proposal.py new file mode 100644 index 0000000000..d6c78cffff --- /dev/null +++ b/hummingbot/strategy/spot_perpetual_arbitrage/arb_proposal.py @@ -0,0 +1,125 @@ +from decimal import Decimal +from hummingbot.core.utils.async_utils import safe_gather +from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple + +s_decimal_nan = Decimal("NaN") +s_decimal_0 = Decimal("0") + + +class ArbProposalSide: + """ + An arbitrage proposal side which contains info needed for order submission. + """ + def __init__(self, + market_info: MarketTradingPairTuple, + is_buy: bool, + order_price: Decimal, + amount: Decimal + ): + """ + :param market_info: The market where to submit the order + :param is_buy: True if buy order + :param quote_price: The quote price (for an order amount) from the market + :param order_price: The price required for order submission, this could differ from the quote price + :param amount: The order amount + """ + self.market_info: MarketTradingPairTuple = market_info + self.is_buy: bool = is_buy + self.order_price: Decimal = order_price + self.amount: Decimal = amount + + def __repr__(self): + side = "Buy" if self.is_buy else "Sell" + base, quote = self.market_info.trading_pair.split("-") + return f"{self.market_info.market.display_name.capitalize()}: {side} {self.amount} {base}" \ + f" at {self.order_price} {quote}." + + +class ArbProposal: + """ + An arbitrage proposal which contains 2 sides of the proposal - one buy and one sell. + """ + def __init__(self, + spot_market_info: MarketTradingPairTuple, + derivative_market_info: MarketTradingPairTuple, + order_amount: Decimal, + timestamp: float): + self.spot_market_info: MarketTradingPairTuple = spot_market_info + self.derivative_market_info: MarketTradingPairTuple = derivative_market_info + self.spot_side: ArbProposalSide = None + self.derivative_side: ArbProposalSide = None + self.amount: Decimal = order_amount + self.timestamp: float = timestamp + self.spot_buy_sell_prices = [0, 0] + self.deriv_buy_sell_prices = [0, 0] + + async def update_prices(self): + """ + Update the buy and sell prices for both spot and deriv connectors. + """ + tasks = [self.spot_market_info.market.get_order_price(self.spot_market_info.trading_pair, True, self.amount), + self.spot_market_info.market.get_order_price(self.spot_market_info.trading_pair, False, self.amount), + self.derivative_market_info.market.get_order_price(self.derivative_market_info.trading_pair, True, self.amount), + self.derivative_market_info.market.get_order_price(self.derivative_market_info.trading_pair, False, self.amount)] + + prices = await safe_gather(*tasks, return_exceptions=True) + self.spot_buy_sell_prices = [prices[0], prices[1]] + self.deriv_buy_sell_prices = [prices[2], prices[3]] + + def is_funding_payment_time(self): + """ + Check if it's time for funding payment. + Return True if it's time for funding payment else False. + """ + funding_info = self.derivative_market_info.market.get_funding_info(self.derivative_market_info.trading_pair) + funding_payment_span = self.derivative_market_info.market._funding_payment_span + if self.timestamp > (funding_info["nextFundingTime"] - funding_payment_span[0]) and \ + self.timestamp < (funding_info["nextFundingTime"] + funding_payment_span[1]): + return True + else: + return False + + async def proposed_spot_deriv_arb(self): + """ + Determine if the current situation is contango or backwardation and return a pair of buy and sell prices accordingly. + """ + await self.update_prices() + if (sum(self.spot_buy_sell_prices) / 2) > (sum(self.deriv_buy_sell_prices) / 2): # Backwardation + self.spot_side = ArbProposalSide(self.spot_market_info, False, + self.spot_buy_sell_prices[1], + self.amount) + self.derivative_side = ArbProposalSide(self.derivative_market_info, True, + self.deriv_buy_sell_prices[0], + self.amount) + return (self.spot_side, self.derivative_side) + else: # Contango + self.spot_side = ArbProposalSide(self.spot_market_info, True, + self.spot_buy_sell_prices[0], + self.amount) + self.derivative_side = ArbProposalSide(self.derivative_market_info, False, + self.deriv_buy_sell_prices[1], + self.amount) + return (self.spot_side, self.derivative_side) + + def alternate_proposal_sides(self): + """ + Alternate the sides and prices of proposed spot and derivative arb. + """ + if self.spot_side.is_buy: + self.spot_side.is_buy = False + self.spot_side.order_price = self.spot_buy_sell_prices[1] + self.derivative_side.is_buy = True + self.derivative_side.order_price = self.deriv_buy_sell_prices[0] + else: + self.spot_side.is_buy = True + self.spot_side.order_price = self.spot_buy_sell_prices[0] + self.derivative_side.is_buy = False + self.derivative_side.order_price = self.deriv_buy_sell_prices[1] + return (self.spot_side, self.derivative_side) + + def spread(self): + spread = abs(self.spot_side.order_price - self.derivative_side.order_price) / min(self.spot_side.order_price, self.derivative_side.order_price) + return Decimal(str(spread)) + + def __repr__(self): + return f"Spot - {self.spot_market_info.market}\nDerivative - {self.derivative_market_info.market}" diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/dummy.pxd b/hummingbot/strategy/spot_perpetual_arbitrage/dummy.pxd new file mode 100644 index 0000000000..4b098d6f59 --- /dev/null +++ b/hummingbot/strategy/spot_perpetual_arbitrage/dummy.pxd @@ -0,0 +1,2 @@ +cdef class dummy(): + pass diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/dummy.pyx b/hummingbot/strategy/spot_perpetual_arbitrage/dummy.pyx new file mode 100644 index 0000000000..4b098d6f59 --- /dev/null +++ b/hummingbot/strategy/spot_perpetual_arbitrage/dummy.pyx @@ -0,0 +1,2 @@ +cdef class dummy(): + pass diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py new file mode 100644 index 0000000000..3ae784fe62 --- /dev/null +++ b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py @@ -0,0 +1,497 @@ +from decimal import Decimal +import time +import logging +import asyncio +import pandas as pd +from typing import List, Dict, Tuple +from hummingbot.core.utils.async_utils import safe_ensure_future +from hummingbot.core.clock import Clock +from hummingbot.core.data_type.limit_order import LimitOrder +from hummingbot.core.data_type.market_order import MarketOrder +from hummingbot.logger import HummingbotLogger +from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple +from hummingbot.strategy.strategy_py_base import StrategyPyBase +from hummingbot.connector.connector_base import ConnectorBase + +from hummingbot.core.event.events import ( + PositionAction, + PositionSide, + PositionMode +) +from hummingbot.connector.derivative.position import Position + +from .arb_proposal import ArbProposalSide, ArbProposal + + +NaN = float("nan") +s_decimal_zero = Decimal(0) +spa_logger = None + + +class SpotPerpetualArbitrageStrategy(StrategyPyBase): + """ + This strategy arbitrages between a spot and a perpetual exchange connector. + For a given order amount, the strategy checks the divergence and convergence in prices that could occur + before and during funding payment on the perpetual exchange. + If presents, the strategy submits taker orders to both market. + """ + + @classmethod + def logger(cls) -> HummingbotLogger: + global spa_logger + if spa_logger is None: + spa_logger = logging.getLogger(__name__) + return spa_logger + + def __init__(self, + spot_market_info: MarketTradingPairTuple, + derivative_market_info: MarketTradingPairTuple, + order_amount: Decimal, + derivative_leverage: int, + min_divergence: Decimal, + min_convergence: Decimal, + spot_market_slippage_buffer: Decimal = Decimal("0"), + derivative_market_slippage_buffer: Decimal = Decimal("0"), + maximize_funding_rate: bool = True, + status_report_interval: float = 10): + """ + :param spot_market_info: The first market + :param derivative_market_info: The second market + :param order_amount: The order amount + :param min_divergence: The minimum spread to start arbitrage (e.g. 0.0003 for 0.3%) + :param min_convergence: The minimum spread to close arbitrage (e.g. 0.0003 for 0.3%) + :param spot_market_slippage_buffer: The buffer for which to adjust order price for higher chance of + the order getting filled. This is quite important for AMM which transaction takes a long time where a slippage + is acceptable rather having the transaction get rejected. The submitted order price will be adjust higher + for buy order and lower for sell order. + :param derivative_market_slippage_buffer: The slipper buffer for market_2 + :param maximize_funding_rate: whether to submit both arbitrage taker orders (buy and sell) simultaneously + If false, the bot will wait for first exchange order filled before submitting the other order. + """ + super().__init__() + self._spot_market_info = spot_market_info + self._derivative_market_info = derivative_market_info + self._min_divergence = min_divergence + self._min_convergence = min_convergence + self._order_amount = order_amount + self._derivative_leverage = derivative_leverage + self._spot_market_slippage_buffer = spot_market_slippage_buffer + self._derivative_market_slippage_buffer = derivative_market_slippage_buffer + self._maximize_funding_rate = maximize_funding_rate + self._all_markets_ready = False + + self._ev_loop = asyncio.get_event_loop() + + self._last_timestamp = 0 + self._status_report_interval = status_report_interval + self.add_markets([spot_market_info.market, derivative_market_info.market]) + + self._current_proposal = None + self._main_task = None + self._spot_done = True + self._deriv_done = True + self._spot_order_ids = [] + self._deriv_order_ids = [] + + @property + def current_proposal(self) -> ArbProposal: + return self._current_proposal + + @current_proposal.setter + def current_proposal(self, value): + self._current_proposal = value + + @property + def min_divergence(self) -> Decimal: + return self._min_divergence + + @property + def min_convergence(self) -> Decimal: + return self._min_convergence + + @property + def order_amount(self) -> Decimal: + return self._order_amount + + @order_amount.setter + def order_amount(self, value): + self._order_amount = value + + @property + def market_info_to_active_orders(self) -> Dict[MarketTradingPairTuple, List[LimitOrder]]: + return self._sb_order_tracker.market_pair_to_active_orders + + @property + def deriv_position(self) -> List[Position]: + return [s for s in self._derivative_market_info.market._account_positions.values() if + s.trading_pair == self._derivative_market_info.trading_pair] + + def tick(self, timestamp: float): + """ + Clock tick entry point, is run every second (on normal tick setting). + :param timestamp: current tick timestamp + """ + if not self._all_markets_ready: + self._all_markets_ready = all([market.ready for market in self.active_markets]) + if not self._all_markets_ready: + # self.logger().warning("Markets are not ready. Please wait...") + return + else: + self.logger().info("Markets are ready. Trading started.") + if len(self.deriv_position) > 0: + self.logger().info("Active position detected, bot assumes first arbitrage was done and would scan for second arbitrage.") + if self.ready_for_new_arb_trades(): + if self._main_task is None or self._main_task.done(): + self.current_proposal = ArbProposal(self._spot_market_info, self._derivative_market_info, self.order_amount, timestamp) + self._main_task = safe_ensure_future(self.main(timestamp)) + + async def main(self, timestamp): + """ + The main procedure for the arbitrage strategy. It first check if it's time for funding payment, decide if to compare with either + min_convergence or min_divergence, applies the slippage buffer, applies budget constraint, then finally execute the + arbitrage. + """ + execute_arb = False + funding_msg = "" + await self.current_proposal.proposed_spot_deriv_arb() + if len(self.deriv_position) > 0 and self.should_alternate_proposal_sides(self.current_proposal, self.deriv_position): + self.current_proposal.alternate_proposal_sides() + + if self.current_proposal.is_funding_payment_time(): + if len(self.deriv_position) > 0: + if self._maximize_funding_rate: + execute_arb = not self.would_receive_funding_payment(self.deriv_position) + if execute_arb: + self.timed_logger(timestamp, "Waiting for funding payment.") + else: + funding_msg = "Time for funding payment, executing second arbitrage to prevent paying funding fee" + else: + funding_msg = "Time for funding payment, executing second arbitrage " \ + "immediately since we don't intend to maximize funding rate" + execute_arb = True + else: + if len(self.deriv_position) > 0: + execute_arb = self.ready_for_execution(self.current_proposal, False) + else: + execute_arb = self.ready_for_execution(self.current_proposal, True) + + if execute_arb: + self.logger().info(self.spread_msg()) + self.apply_slippage_buffers(self.current_proposal) + self.apply_budget_constraint(self.current_proposal) + await self.execute_arb_proposals(self.current_proposal, funding_msg) + else: + self.timed_logger(timestamp, self.spread_msg()) + return + + def timed_logger(self, timestamp, msg): + """ + Displays log at specific intervals. + :param timestamp: current timestamp + :param msg: message to display at next interval + """ + if timestamp - self._last_timestamp > self._status_report_interval: + self.logger().info(msg) + self._last_timestamp = timestamp + + def ready_for_execution(self, proposal: ArbProposal, first: bool): + """ + Check if the spread meets the required spread requirement for the right arbitrage. + :param proposal: current proposal object + :param first: True, if scanning for opportunity for first arbitrage, else, False + :return: True if ready, else, False + """ + spread = self.current_proposal.spread() + if first and spread >= self.min_divergence: + return True + elif not first and spread <= self.min_convergence: + return True + return False + + def should_alternate_proposal_sides(self, proposal: ArbProposal, active_position: List[Position]): + """ + Checks if there's need to alternate the sides of a proposed arbitrage. + :param proposal: current proposal object + :param active_position: information about active position for the derivative connector + :return: True if sides need to be alternated, else, False + """ + deriv_proposal_side = PositionSide.LONG if proposal.derivative_side.is_buy else PositionSide.SHORT + position_side = PositionSide.LONG if active_position[0].amount > 0 else PositionSide.SHORT + if deriv_proposal_side == position_side: + return True + return False + + def would_receive_funding_payment(self, active_position: List[Position]): + """ + Checks if an active position would receive funding payment. + :param active_position: information about active position for the derivative connector + :return: True if funding payment would be received, else, False + """ + funding_info = self._derivative_market_info.market.get_funding_info(self._derivative_market_info.trading_pair) + if (active_position[0].amount > 0 and funding_info["rate"] < 0) or \ + (active_position[0].amount < 0 and funding_info["rate"] > 0): + return True + return False + + def apply_slippage_buffers(self, arb_proposal: ArbProposal): + """ + Updates arb_proposals by adjusting order price for slipper buffer percentage. + E.g. if it is a buy order, for an order price of 100 and 1% slipper buffer, the new order price is 101, + for a sell order, the new order price is 99. + :param arb_proposals: the arbitrage proposal + """ + for arb_side in (arb_proposal.spot_side, arb_proposal.derivative_side): + market = arb_side.market_info.market + arb_side.amount = market.quantize_order_amount(arb_side.market_info.trading_pair, arb_side.amount) + s_buffer = self._spot_market_slippage_buffer if market == self._spot_market_info.market \ + else self._derivative_market_slippage_buffer + if not arb_side.is_buy: + s_buffer *= Decimal("-1") + arb_side.order_price *= Decimal("1") + s_buffer + arb_side.order_price = market.quantize_order_price(arb_side.market_info.trading_pair, + arb_side.order_price) + + def apply_budget_constraint(self, arb_proposal: ArbProposal): + """ + Updates arb_proposals by setting proposal amount to 0 if there is not enough balance to submit order with + required order amount. + :param arb_proposals: the arbitrage proposal + """ + spot_market = self._spot_market_info.market + deriv_market = self._derivative_market_info.market + spot_token = self._spot_market_info.quote_asset if arb_proposal.spot_side.is_buy else self._spot_market_info.base_asset + deriv_token = self._derivative_market_info.quote_asset + spot_token_balance = spot_market.get_available_balance(spot_token) + deriv_token_balance = deriv_market.get_available_balance(deriv_token) + required_spot_balance = arb_proposal.amount * arb_proposal.spot_side.order_price if arb_proposal.spot_side.is_buy else arb_proposal.amount + required_deriv_balance = arb_proposal.amount * arb_proposal.derivative_side.order_price + if spot_token_balance < required_spot_balance: + arb_proposal.amount = s_decimal_zero + self.logger().info(f"Can't arbitrage, {spot_market.display_name} " + f"{spot_token} balance " + f"({spot_token_balance}) is below required order amount ({required_spot_balance}).") + elif deriv_token_balance < required_deriv_balance: + arb_proposal.amount = s_decimal_zero + self.logger().info(f"Can't arbitrage, {deriv_market.display_name} " + f"{deriv_token} balance " + f"({deriv_token_balance}) is below required order amount ({required_deriv_balance}).") + return + + async def execute_arb_proposals(self, arb_proposal: ArbProposal, is_funding_msg: str = ""): + """ + Execute both sides of the arbitrage trades concurrently. + :param arb_proposals: the arbitrage proposal + :param is_funding_msg: message pertaining to funding payment + """ + if arb_proposal.amount == s_decimal_zero: + return + self._spot_done = False + self._deriv_done = False + proposal = self.short_proposal_msg(False) + if is_funding_msg: + opportunity_msg = is_funding_msg + else: + first_arbitage = not bool(len(self.deriv_position)) + opportunity_msg = "Spread wide enough to execute first arbitrage" if first_arbitage else \ + "Spread low enough to execute second arbitrage" + self.logger().info(f"{opportunity_msg}!: \n" + f"{proposal[0]} \n" + f"{proposal[1]} \n") + safe_ensure_future(self.execute_spot_side(arb_proposal.spot_side)) + safe_ensure_future(self.execute_derivative_side(arb_proposal.derivative_side)) + + async def execute_spot_side(self, arb_side: ArbProposalSide): + side = "BUY" if arb_side.is_buy else "SELL" + place_order_fn = self.buy_with_specific_market if arb_side.is_buy else self.sell_with_specific_market + self.log_with_clock(logging.INFO, + f"Placing {side} order for {arb_side.amount} {arb_side.market_info.base_asset} " + f"at {arb_side.market_info.market.display_name} at {arb_side.order_price} price") + order_id = place_order_fn(arb_side.market_info, + arb_side.amount, + arb_side.market_info.market.get_taker_order_type(), + arb_side.order_price, + ) + self._spot_order_ids.append(order_id) + + async def execute_derivative_side(self, arb_side: ArbProposalSide): + side = "BUY" if arb_side.is_buy else "SELL" + place_order_fn = self.buy_with_specific_market if arb_side.is_buy else self.sell_with_specific_market + position_action = PositionAction.OPEN if len(self.deriv_position) == 0 else PositionAction.CLOSE + self.log_with_clock(logging.INFO, + f"Placing {side} order for {arb_side.amount} {arb_side.market_info.base_asset} " + f"at {arb_side.market_info.market.display_name} at {arb_side.order_price} price to {position_action.name} position.") + order_id = place_order_fn(arb_side.market_info, + arb_side.amount, + arb_side.market_info.market.get_taker_order_type(), + arb_side.order_price, + position_action=position_action + ) + self._deriv_order_ids.append(order_id) + + def ready_for_new_arb_trades(self) -> bool: + """ + Returns True if there is no outstanding unfilled order. + """ + for market_info in [self._spot_market_info, self._derivative_market_info]: + if len(self.market_info_to_active_orders.get(market_info, [])) > 0: + return False + if not self._spot_done or not self._deriv_done: + return False + return True + + def short_proposal_msg(self, indented: bool = True) -> List[str]: + """ + Composes a short proposal message. + :param indented: If the message should be indented (by 4 spaces) + :return A list of info on both sides of an arbitrage + """ + lines = [] + proposal = self.current_proposal + lines.append(f"{' ' if indented else ''}{proposal.spot_side}") + lines.append(f"{' ' if indented else ''}{proposal.derivative_side}") + return lines + + def spread_msg(self): + """ + Composes a short spread message. + :return Info about current spread of an arbitrage + """ + spread = self.current_proposal.spread() + first = not bool(len(self.deriv_position)) + target_spread_str = "minimum divergence spread" if first else "minimum convergence spread" + target_spread = self.min_divergence if first else self.min_convergence + msg = f"Current spread: {spread:.2%}, {target_spread_str}: {target_spread:.2%}." + return msg + + def active_positions_df(self) -> pd.DataFrame: + columns = ["Symbol", "Type", "Entry Price", "Amount", "Leverage", "Unrealized PnL"] + data = [] + market, trading_pair = self._derivative_market_info.market, self._derivative_market_info.trading_pair + for idx in self.deriv_position: + is_buy = True if idx.amount > 0 else False + unrealized_profit = ((market.get_price(trading_pair, is_buy) - idx.entry_price) * idx.amount) + data.append([ + idx.trading_pair, + idx.position_side.name, + idx.entry_price, + idx.amount, + idx.leverage, + unrealized_profit + ]) + + return pd.DataFrame(data=data, columns=columns) + + async def format_status(self) -> str: + """ + Returns a status string formatted to display nicely on terminal. The strings composes of 4 parts: markets, + assets, spread and warnings(if any). + """ + columns = ["Exchange", "Market", "Sell Price", "Buy Price", "Mid Price"] + data = [] + for market_info in [self._spot_market_info, self._derivative_market_info]: + market, trading_pair, base_asset, quote_asset = market_info + buy_price = await market.get_quote_price(trading_pair, True, self._order_amount) + sell_price = await market.get_quote_price(trading_pair, False, self._order_amount) + mid_price = (buy_price + sell_price) / 2 + data.append([ + market.display_name, + trading_pair, + float(sell_price), + float(buy_price), + float(mid_price) + ]) + markets_df = pd.DataFrame(data=data, columns=columns) + lines = [] + lines.extend(["", " Markets:"] + [" " + line for line in markets_df.to_string(index=False).split("\n")]) + + # See if there're any active positions. + if len(self.deriv_position) > 0: + df = self.active_positions_df() + lines.extend(["", " Positions:"] + [" " + line for line in df.to_string(index=False).split("\n")]) + else: + lines.extend(["", " No active positions."]) + + assets_df = self.wallet_balance_data_frame([self._spot_market_info, self._derivative_market_info]) + lines.extend(["", " Assets:"] + + [" " + line for line in str(assets_df).split("\n")]) + + lines.extend(["", " Spread details:"] + [" " + self.spread_msg()] + + self.short_proposal_msg()) + + warning_lines = self.network_warning([self._spot_market_info]) + warning_lines.extend(self.network_warning([self._derivative_market_info])) + warning_lines.extend(self.balance_warning([self._spot_market_info])) + warning_lines.extend(self.balance_warning([self._derivative_market_info])) + if len(warning_lines) > 0: + lines.extend(["", "*** WARNINGS ***"] + warning_lines) + + return "\n".join(lines) + + def did_complete_buy_order(self, order_completed_event): + self.update_status(order_completed_event) + + def did_complete_sell_order(self, order_completed_event): + self.update_status(order_completed_event) + + def did_fail_order(self, order_failed_event): + self.retry_order(order_failed_event) + + def did_cancel_order(self, cancelled_event): + self.retry_order(cancelled_event) + + def did_expire_order(self, expired_event): + self.retry_order(expired_event) + + def did_complete_funding_payment(self, funding_payment_completed_event): + # Excute second arbitrage if necessary (even spread hasn't reached min convergence) + if len(self.deriv_position) > 0: + self.apply_slippage_buffers(self.current_proposal) + self.apply_budget_constraint(self.current_proposal) + funding_msg = "Executing second arbitrage after funding payment is received" + safe_ensure_future(self.execute_arb_proposals(self.current_proposal, funding_msg)) + return + + def update_status(self, event): + order_id = event.order_id + if order_id in self._spot_order_ids: + self._spot_done = True + self._spot_order_ids.remove(order_id) + elif order_id in self._deriv_order_ids: + self._deriv_done = True + self._deriv_order_ids.remove(order_id) + + def retry_order(self, event): + order_id = event.order_id + # To-do: Should be updated to do counted retry rather than time base retry. i.e mark as done after retrying 3 times + if event.timestamp > (time.time() - 5): # retry if order failed less than 5 secs ago + if order_id in self._spot_order_ids: + self.logger().info("Retrying failed order on spot exchange.") + safe_ensure_future(self.execute_spot_side(self.current_proposal.spot_side)) + self._spot_order_ids.remove(order_id) + elif order_id in self._deriv_order_ids: + self.logger().info("Retrying failed order on derivative exchange.") + safe_ensure_future(self.execute_derivative_side(self.current_proposal.derivative_side)) + self._deriv_order_ids.remove(order_id) + else: # mark as done + self.update_status(event) + + @property + def tracked_limit_orders(self) -> List[Tuple[ConnectorBase, LimitOrder]]: + return self._sb_order_tracker.tracked_limit_orders + + @property + def tracked_market_orders(self) -> List[Tuple[ConnectorBase, MarketOrder]]: + return self._sb_order_tracker.tracked_market_orders + + def apply_initial_settings(self, trading_pair, leverage): + deriv_market = self._derivative_market_info.market + deriv_market.set_leverage(trading_pair, leverage) + deriv_market.set_position_mode(PositionMode.ONEWAY) + + def start(self, clock: Clock, timestamp: float): + self.apply_initial_settings(self._derivative_market_info.trading_pair, self._derivative_leverage) + + def stop(self, clock: Clock): + if self._main_task is not None: + self._main_task.cancel() + self._main_task = None diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py new file mode 100644 index 0000000000..8be3cd9653 --- /dev/null +++ b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py @@ -0,0 +1,125 @@ +from hummingbot.client.config.config_var import ConfigVar +from hummingbot.client.config.config_validators import ( + validate_connector, + validate_derivative, + validate_decimal, + validate_bool, + validate_int +) +from hummingbot.client.settings import ( + required_exchanges, + requried_connector_trading_pairs, + EXAMPLE_PAIRS, +) +from decimal import Decimal + + +def exchange_on_validated(value: str) -> None: + required_exchanges.append(value) + + +def spot_market_on_validated(value: str) -> None: + requried_connector_trading_pairs[spot_perpetual_arbitrage_config_map["spot_connector"].value] = [value] + + +def derivative_market_on_validated(value: str) -> None: + requried_connector_trading_pairs[spot_perpetual_arbitrage_config_map["derivative_connector"].value] = [value] + + +def spot_market_prompt() -> str: + connector = spot_perpetual_arbitrage_config_map.get("spot_connector").value + example = EXAMPLE_PAIRS.get(connector) + return "Enter the token trading pair you would like to trade on %s%s >>> " \ + % (connector, f" (e.g. {example})" if example else "") + + +def derivative_market_prompt() -> str: + connector = spot_perpetual_arbitrage_config_map.get("derivative_connector").value + example = EXAMPLE_PAIRS.get(connector) + return "Enter the token trading pair you would like to trade on %s%s >>> " \ + % (connector, f" (e.g. {example})" if example else "") + + +def order_amount_prompt() -> str: + trading_pair = spot_perpetual_arbitrage_config_map["spot_market"].value + base_asset, quote_asset = trading_pair.split("-") + return f"What is the amount of {base_asset} per order? >>> " + + +spot_perpetual_arbitrage_config_map = { + "strategy": ConfigVar( + key="strategy", + prompt="", + default="spot_perpetual_arbitrage"), + "spot_connector": ConfigVar( + key="connector_1", + prompt="Enter a spot connector (Exchange/AMM) >>> ", + prompt_on_new=True, + validator=validate_connector, + on_validated=exchange_on_validated), + "spot_market": ConfigVar( + key="spot_market", + prompt=spot_market_prompt, + prompt_on_new=True, + on_validated=spot_market_on_validated), + "derivative_connector": ConfigVar( + key="derivative_connector", + prompt="Enter a derivative name (Exchange/AMM) >>> ", + prompt_on_new=True, + validator=validate_derivative, + on_validated=exchange_on_validated), + "derivative_market": ConfigVar( + key="derivative_market", + prompt=derivative_market_prompt, + prompt_on_new=True, + on_validated=derivative_market_on_validated), + "order_amount": ConfigVar( + key="order_amount", + prompt=order_amount_prompt, + type_str="decimal", + prompt_on_new=True), + "derivative_leverage": ConfigVar( + key="derivative_leverage", + prompt="How much leverage would you like to use on the derivative exchange? (Enter 1 to indicate 1X) ", + type_str="int", + default=1, + validator= lambda v: validate_int(v), + prompt_on_new=True), + "min_divergence": ConfigVar( + key="min_divergence", + prompt="What is the minimum spread between the spot and derivative market price before starting an arbitrage? (Enter 1 to indicate 1%) >>> ", + prompt_on_new=True, + default=Decimal("1"), + validator=lambda v: validate_decimal(v, 0, 100, inclusive=False), + type_str="decimal"), + "min_convergence": ConfigVar( + key="min_convergence", + prompt="What is the minimum spread between the spot and derivative market price before closing an existing arbitrage? (Enter 1 to indicate 1%) >>> ", + prompt_on_new=True, + default=Decimal("1"), + validator=lambda v: validate_decimal(v, 0, spot_perpetual_arbitrage_config_map["min_divergence"].value), + type_str="decimal"), + "maximize_funding_rate": ConfigVar( + key="maximize_funding_rate", + prompt="Would you like to take advantage of the funding rate on the derivative exchange, even if min convergence is reached during funding time? (True/False) >>> ", + prompt_on_new=True, + default=False, + validator=validate_bool, + type_str="bool"), + "spot_market_slippage_buffer": ConfigVar( + key="spot_market_slippage_buffer", + prompt="How much buffer do you want to add to the price to account for slippage for orders on the spot market " + "(Enter 1 for 1%)? >>> ", + prompt_on_new=True, + default=Decimal("0.05"), + validator=lambda v: validate_decimal(v), + type_str="decimal"), + "derivative_market_slippage_buffer": ConfigVar( + key="derivative_market_slippage_buffer", + prompt="How much buffer do you want to add to the price to account for slippage for orders on the derivative market" + " (Enter 1 for 1%)? >>> ", + prompt_on_new=True, + default=Decimal("0.05"), + validator=lambda v: validate_decimal(v), + type_str="decimal"), +} diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/start.py b/hummingbot/strategy/spot_perpetual_arbitrage/start.py new file mode 100644 index 0000000000..2f145f7dde --- /dev/null +++ b/hummingbot/strategy/spot_perpetual_arbitrage/start.py @@ -0,0 +1,37 @@ +from decimal import Decimal +from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple +from hummingbot.strategy.spot_perpetual_arbitrage.spot_perpetual_arbitrage import SpotPerpetualArbitrageStrategy +from hummingbot.strategy.spot_perpetual_arbitrage.spot_perpetual_arbitrage_config_map import spot_perpetual_arbitrage_config_map + + +def start(self): + spot_connector = spot_perpetual_arbitrage_config_map.get("spot_connector").value.lower() + spot_market = spot_perpetual_arbitrage_config_map.get("spot_market").value + derivative_connector = spot_perpetual_arbitrage_config_map.get("derivative_connector").value.lower() + derivative_market = spot_perpetual_arbitrage_config_map.get("derivative_market").value + order_amount = spot_perpetual_arbitrage_config_map.get("order_amount").value + derivative_leverage = spot_perpetual_arbitrage_config_map.get("derivative_leverage").value + min_divergence = spot_perpetual_arbitrage_config_map.get("min_divergence").value / Decimal("100") + min_convergence = spot_perpetual_arbitrage_config_map.get("min_convergence").value / Decimal("100") + spot_market_slippage_buffer = spot_perpetual_arbitrage_config_map.get("spot_market_slippage_buffer").value / Decimal("100") + derivative_market_slippage_buffer = spot_perpetual_arbitrage_config_map.get("derivative_market_slippage_buffer").value / Decimal("100") + maximize_funding_rate = spot_perpetual_arbitrage_config_map.get("maximize_funding_rate").value + + self._initialize_markets([(spot_connector, [spot_market]), (derivative_connector, [derivative_market])]) + base_1, quote_1 = spot_market.split("-") + base_2, quote_2 = derivative_market.split("-") + self.assets = set([base_1, quote_1, base_2, quote_2]) + + spot_market_info = MarketTradingPairTuple(self.markets[spot_connector], spot_market, base_1, quote_1) + derivative_market_info = MarketTradingPairTuple(self.markets[derivative_connector], derivative_market, base_2, quote_2) + + self.market_trading_pair_tuples = [spot_market_info, derivative_market_info] + self.strategy = SpotPerpetualArbitrageStrategy(spot_market_info, + derivative_market_info, + order_amount, + derivative_leverage, + min_divergence, + min_convergence, + spot_market_slippage_buffer, + derivative_market_slippage_buffer, + maximize_funding_rate) diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/utils.py b/hummingbot/strategy/spot_perpetual_arbitrage/utils.py new file mode 100644 index 0000000000..f5b2e39936 --- /dev/null +++ b/hummingbot/strategy/spot_perpetual_arbitrage/utils.py @@ -0,0 +1,44 @@ +from decimal import Decimal +from typing import List +from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple +from .data_types import ArbProposal, ArbProposalSide + +s_decimal_nan = Decimal("NaN") + + +async def create_arb_proposals(market_info_1: MarketTradingPairTuple, + market_info_2: MarketTradingPairTuple, + order_amount: Decimal) -> List[ArbProposal]: + """ + Creates base arbitrage proposals for given markets without any filtering. + :param market_info_1: The first market + :param market_info_2: The second market + :param order_amount: The required order amount. + :return A list of 2 proposal - (market_1 buy, market_2 sell) and (market_1 sell, market_2 buy) + """ + order_amount = Decimal(str(order_amount)) + results = [] + for index in range(0, 2): + is_buy = not bool(index) # bool(0) is False, so start with buy first + m_1_q_price = await market_info_1.market.get_quote_price(market_info_1.trading_pair, is_buy, order_amount) + m_1_o_price = await market_info_1.market.get_order_price(market_info_1.trading_pair, is_buy, order_amount) + m_2_q_price = await market_info_2.market.get_quote_price(market_info_2.trading_pair, not is_buy, order_amount) + m_2_o_price = await market_info_2.market.get_order_price(market_info_2.trading_pair, not is_buy, order_amount) + if any(p is None for p in (m_1_o_price, m_1_q_price, m_2_o_price, m_2_q_price)): + continue + first_side = ArbProposalSide( + market_info_1, + is_buy, + m_1_q_price, + m_1_o_price, + order_amount + ) + second_side = ArbProposalSide( + market_info_2, + not is_buy, + m_2_q_price, + m_2_o_price, + order_amount + ) + results.append(ArbProposal(first_side, second_side)) + return results diff --git a/hummingbot/templates/conf_spot_perpetual_arbitrage_strategy_TEMPLATE.yml b/hummingbot/templates/conf_spot_perpetual_arbitrage_strategy_TEMPLATE.yml new file mode 100644 index 0000000000..7945f553c4 --- /dev/null +++ b/hummingbot/templates/conf_spot_perpetual_arbitrage_strategy_TEMPLATE.yml @@ -0,0 +1,37 @@ +########################################## +### Spot-Perpetual Arbitrage strategy config ### +########################################## + +template_version: 1 +strategy: null + +# The following configuations are only required for the AMM arbitrage trading strategy + +# Connectors and markets parameters +spot_connector: null +spot_market: null +derivative_connector: null +derivative_market: null + +order_amount: null + +derivative_leverage: null + +# Spread required before first arbitrage can take place, expressed in percentage value, e.g. 1 = 1% +min_divergence: null + +# Spread required after first arbitrage before second arbitreage can take place, expressed in percentage value, e.g. 1 = 1% +min_convergence: null + +# A buffer for which to adjust order price for higher chance of the order getting filled. +# This is important for AMM which transaction takes a long time where a slippage is acceptable rather having +# the transaction get rejected. The submitted order price will be adjust higher (by percentage value) for buy order +# and lower for sell order. (Enter 1 for 1%) +spot_market_slippage_buffer: null + +# A buffer to add to the price to account for slippage when buying/selling on second connector market +# (Enter 1 for 1%) +derivative_market_slippage_buffer: null + +# A flag (true/false), if true the bot would only execute second arbitrage once funding payment is received provided second arbitrage wasn't executed before funding period. +maximize_funding_rate: null From ee58e907fced06946a3d5fcc76205d90a0533b59 Mon Sep 17 00:00:00 2001 From: Nullably <37262506+Nullably@users.noreply.github.com> Date: Tue, 23 Feb 2021 16:14:52 +0800 Subject: [PATCH 41/73] (feat) update status and add max_spread and max_order_age --- .../liquidity_mining/liquidity_mining.py | 76 +++++++++++++------ .../liquidity_mining_config_map.py | 12 +++ hummingbot/strategy/liquidity_mining/start.py | 4 + ...onf_liquidity_mining_strategy_TEMPLATE.yml | 8 +- 4 files changed, 76 insertions(+), 24 deletions(-) diff --git a/hummingbot/strategy/liquidity_mining/liquidity_mining.py b/hummingbot/strategy/liquidity_mining/liquidity_mining.py index fd511f2721..5b8d38583d 100644 --- a/hummingbot/strategy/liquidity_mining/liquidity_mining.py +++ b/hummingbot/strategy/liquidity_mining/liquidity_mining.py @@ -49,6 +49,8 @@ def __init__(self, volatility_interval: int = 60 * 5, avg_volatility_period: int = 10, volatility_to_spread_multiplier: Decimal = Decimal("1"), + max_spread: Decimal = Decimal("-1"), + max_order_age: float = 60. * 60., status_report_interval: float = 900, hb_app_notification: bool = False): super().__init__() @@ -65,6 +67,8 @@ def __init__(self, self._volatility_interval = volatility_interval self._avg_volatility_period = avg_volatility_period self._volatility_to_spread_multiplier = volatility_to_spread_multiplier + self._max_spread = max_spread + self._max_order_age = max_order_age self._ev_loop = asyncio.get_event_loop() self._last_timestamp = 0 self._status_report_interval = status_report_interval @@ -112,6 +116,12 @@ def tick(self, timestamp: float): self._last_timestamp = timestamp + @staticmethod + def order_age(order: LimitOrder) -> float: + if "//" not in order.client_order_id: + return int(time.time()) - int(order.client_order_id[-16:]) / 1e6 + return -1. + async def active_orders_df(self) -> pd.DataFrame: size_q_col = f"Size ({self._token})" if self.is_token_a_quote_token() else "Size (Quote)" columns = ["Market", "Side", "Price", "Spread", "Amount", size_q_col, "Age"] @@ -120,11 +130,9 @@ async def active_orders_df(self) -> pd.DataFrame: mid_price = self._market_infos[order.trading_pair].get_mid_price() spread = 0 if mid_price == 0 else abs(order.price - mid_price) / mid_price size_q = order.quantity * mid_price - age = "n/a" + age = self.order_age(order) # // indicates order is a paper order so 'n/a'. For real orders, calculate age. - if "//" not in order.client_order_id: - age = pd.Timestamp(int(time.time()) - int(order.client_order_id[-16:]) / 1e6, - unit='s').strftime('%H:%M:%S') + age_txt = "n/a" if age <= 0. else pd.Timestamp(age, unit='s').strftime('%H:%M:%S') data.append([ order.trading_pair, "buy" if order.is_buy else "sell", @@ -132,48 +140,64 @@ async def active_orders_df(self) -> pd.DataFrame: f"{spread:.2%}", float(order.quantity), float(size_q), - age + age_txt ]) return pd.DataFrame(data=data, columns=columns) - def market_status_df(self) -> pd.DataFrame: + def budget_status_df(self) -> pd.DataFrame: data = [] - columns = ["Market", "Mid Price", "Volatility", "Base Bal", "Quote Bal", f"Budget ({self._token})", - "Base %", "Quote %"] + columns = ["Market", f"Budget ({self._token})", "Base Bal", "Quote Bal", "Base / Quote"] for market, market_info in self._market_infos.items(): mid_price = market_info.get_mid_price() base_bal = self._sell_budgets[market] quote_bal = self._buy_budgets[market] - total_bal = (base_bal * mid_price) + quote_bal - total_bal_in_token = total_bal + total_bal_in_quote = (base_bal * mid_price) + quote_bal + total_bal_in_token = total_bal_in_quote if not self.is_token_a_quote_token(): total_bal_in_token = base_bal + (quote_bal / mid_price) - base_pct = (base_bal * mid_price) / total_bal if total_bal > 0 else s_decimal_zero - quote_pct = quote_bal / total_bal if total_bal > 0 else s_decimal_zero + base_pct = (base_bal * mid_price) / total_bal_in_quote if total_bal_in_quote > 0 else s_decimal_zero + quote_pct = quote_bal / total_bal_in_quote if total_bal_in_quote > 0 else s_decimal_zero data.append([ market, - float(mid_price), - "" if self._volatility[market].is_nan() else f"{self._volatility[market]:.2%}", + float(total_bal_in_token), float(base_bal), float(quote_bal), - float(total_bal_in_token), - f"{base_pct:.0%}", - f"{quote_pct:.0%}" + f"{base_pct:.0%} / {quote_pct:.0%}" + ]) + return pd.DataFrame(data=data, columns=columns).replace(np.nan, '', regex=True) + + def market_status_df(self) -> pd.DataFrame: + data = [] + columns = ["Market", "Mid Price", "Best Bid %", "Best Ask %", "Volatility"] + for market, market_info in self._market_infos.items(): + mid_price = market_info.get_mid_price() + best_bid = self._exchange.get_price(market, False) + best_ask = self._exchange.get_price(market, True) + best_bid_pct = abs(best_bid - mid_price) / mid_price + best_ask_pct = (best_ask - mid_price) / mid_price + data.append([ + market, + float(mid_price), + f"{best_bid_pct:.2%}", + f"{best_ask_pct:.2%}", + "" if self._volatility[market].is_nan() else f"{self._volatility[market]:.2%}", ]) return pd.DataFrame(data=data, columns=columns).replace(np.nan, '', regex=True) async def miner_status_df(self) -> pd.DataFrame: data = [] - columns = ["Market", "Reward/day", "Liquidity (bots)", "Yield/day", "Max spread"] + columns = ["Market", "Paid in", "Reward/week", "Curr Liquidity", "APY", "Max Spread"] campaigns = await get_campaign_summary(self._exchange.display_name, list(self._market_infos.keys())) for market, campaign in campaigns.items(): + reward_usd = await usd_value(campaign.payout_asset, campaign.reward_per_day * Decimal("7")) liquidity_usd = await usd_value(market.split('-')[0], campaign.liquidity) data.append([ market, - f"{campaign.reward_per_day:.2f} {campaign.payout_asset}", - f"${liquidity_usd:.0f} ({campaign.active_bots})", - f"{campaign.apy / Decimal(365):.2%}", + campaign.payout_asset, + f"${reward_usd:.0f}", + f"${liquidity_usd:.0f}", + f"{campaign.apy:.2%}", f"{campaign.spread_max:.2%}%" ]) return pd.DataFrame(data=data, columns=columns).replace(np.nan, '', regex=True) @@ -185,12 +209,15 @@ async def format_status(self) -> str: warning_lines = [] warning_lines.extend(self.network_warning(list(self._market_infos.values()))) + budget_df = self.budget_status_df() + lines.extend(["", " Budget:"] + [" " + line for line in budget_df.to_string(index=False).split("\n")]) + market_df = self.market_status_df() lines.extend(["", " Markets:"] + [" " + line for line in market_df.to_string(index=False).split("\n")]) miner_df = await self.miner_status_df() if not miner_df.empty: - lines.extend(["", " Miners:"] + [" " + line for line in miner_df.to_string(index=False).split("\n")]) + lines.extend(["", " Miner:"] + [" " + line for line in miner_df.to_string(index=False).split("\n")]) # See if there're any open orders. if len(self.active_orders) > 0: @@ -219,6 +246,8 @@ def create_base_proposals(self): if not self._volatility[market].is_nan(): # volatility applies only when it is higher than the spread setting. spread = max(spread, self._volatility[market] * self._volatility_to_spread_multiplier) + if self._max_spread > s_decimal_zero: + spread = min(spread, self._max_spread) mid_price = market_info.get_mid_price() buy_price = mid_price * (Decimal("1") - spread) buy_price = self._exchange.quantize_order_price(market, buy_price) @@ -301,7 +330,8 @@ def cancel_active_orders(self, proposals: List[Proposal]): if self._refresh_times[proposal.market] > self.current_timestamp: continue cur_orders = [o for o in self.active_orders if o.trading_pair == proposal.market] - if not cur_orders or self.is_within_tolerance(cur_orders, proposal): + if not cur_orders or (all(self.order_age(o) < self._max_order_age for o in cur_orders) + and self.is_within_tolerance(cur_orders, proposal)): continue for order in cur_orders: self.cancel_order(self._market_infos[proposal.market], order.client_order_id) diff --git a/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py b/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py index ece5a141de..2f70e53cc1 100644 --- a/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py +++ b/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py @@ -119,4 +119,16 @@ def order_size_prompt() -> str: type_str="decimal", validator=lambda v: validate_decimal(v, min_value=0, inclusive=False), default=Decimal("1")), + "max_spread": + ConfigVar(key="max_spread", + prompt="What is the maximum spread? (Enter 1 to indicate 1% or -1 to ignore this setting) >>> ", + type_str="decimal", + validator=lambda v: validate_decimal(v), + default=Decimal("-1")), + "max_order_age": + ConfigVar(key="max_order_age", + prompt="What is the maximum life time of your orders (in seconds)? >>> ", + type_str="float", + validator=lambda v: validate_decimal(v, min_value=0, inclusive=False), + default=60. * 60.), } diff --git a/hummingbot/strategy/liquidity_mining/start.py b/hummingbot/strategy/liquidity_mining/start.py index 03c7445276..933a838b36 100644 --- a/hummingbot/strategy/liquidity_mining/start.py +++ b/hummingbot/strategy/liquidity_mining/start.py @@ -22,6 +22,8 @@ def start(self): volatility_interval = c_map.get("volatility_interval").value avg_volatility_period = c_map.get("avg_volatility_period").value volatility_to_spread_multiplier = c_map.get("volatility_to_spread_multiplier").value + max_spread = c_map.get("max_spread").value / Decimal("100") + max_order_age = c_map.get("max_order_age").value self._initialize_markets([(exchange, markets)]) exchange = self.markets[exchange] @@ -43,5 +45,7 @@ def start(self): volatility_interval=volatility_interval, avg_volatility_period=avg_volatility_period, volatility_to_spread_multiplier=volatility_to_spread_multiplier, + max_spread=max_spread, + max_order_age=max_order_age, hb_app_notification=True ) diff --git a/hummingbot/templates/conf_liquidity_mining_strategy_TEMPLATE.yml b/hummingbot/templates/conf_liquidity_mining_strategy_TEMPLATE.yml index 7daab8c8d4..f97380ad5c 100644 --- a/hummingbot/templates/conf_liquidity_mining_strategy_TEMPLATE.yml +++ b/hummingbot/templates/conf_liquidity_mining_strategy_TEMPLATE.yml @@ -2,7 +2,7 @@ ### Liquidity Mining strategy config ### ######################################################## -template_version: 2 +template_version: 3 strategy: null # The exchange to run this strategy. @@ -49,5 +49,11 @@ avg_volatility_period: null # The multiplier used to convert average volatility to spread, enter 1 for 1 to 1 conversion volatility_to_spread_multiplier: null +# The maximum value for spread, enter 1 to indicate 1% or -1 to ignore this setting +max_spread: null + +# The maximum life time of your orders in seconds +max_order_age: null + # For more detailed information, see: # https://docs.hummingbot.io/strategies/liquidity-mining/#configuration-parameters From f633fd471fe32b15824b8cc3bd67a5f0213e7586 Mon Sep 17 00:00:00 2001 From: vic-en Date: Tue, 23 Feb 2021 15:03:11 +0100 Subject: [PATCH 42/73] (feat) fix failed jenkins tests --- hummingbot/strategy/strategy_base.pxd | 1 + 1 file changed, 1 insertion(+) diff --git a/hummingbot/strategy/strategy_base.pxd b/hummingbot/strategy/strategy_base.pxd index efb35d8b09..0d5683434e 100644 --- a/hummingbot/strategy/strategy_base.pxd +++ b/hummingbot/strategy/strategy_base.pxd @@ -16,6 +16,7 @@ cdef class StrategyBase(TimeIterator): EventListener _sb_expire_order_listener EventListener _sb_complete_buy_order_listener EventListener _sb_complete_sell_order_listener + EventListener _sb_complete_funding_payment_listener bint _sb_delegate_lock public OrderTracker _sb_order_tracker From 6d8f617c0af58dcd3fb1f2c721d88c161cca7d3d Mon Sep 17 00:00:00 2001 From: Nullably <37262506+Nullably@users.noreply.github.com> Date: Wed, 24 Feb 2021 12:44:15 +0800 Subject: [PATCH 43/73] (feat) update cancel priority where max_order_age is first --- .../liquidity_mining/liquidity_mining.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/hummingbot/strategy/liquidity_mining/liquidity_mining.py b/hummingbot/strategy/liquidity_mining/liquidity_mining.py index 5b8d38583d..c681ce299e 100644 --- a/hummingbot/strategy/liquidity_mining/liquidity_mining.py +++ b/hummingbot/strategy/liquidity_mining/liquidity_mining.py @@ -327,16 +327,20 @@ def is_within_tolerance(self, cur_orders: List[LimitOrder], proposal: Proposal): def cancel_active_orders(self, proposals: List[Proposal]): for proposal in proposals: - if self._refresh_times[proposal.market] > self.current_timestamp: - continue + to_cancel = True cur_orders = [o for o in self.active_orders if o.trading_pair == proposal.market] - if not cur_orders or (all(self.order_age(o) < self._max_order_age for o in cur_orders) - and self.is_within_tolerance(cur_orders, proposal)): - continue - for order in cur_orders: - self.cancel_order(self._market_infos[proposal.market], order.client_order_id) - # To place new order on the next tick - self._refresh_times[order.trading_pair] = self.current_timestamp + 0.1 + if cur_orders and any(self.order_age(o) > self._max_order_age for o in cur_orders): + to_cancel = True + else: + if self._refresh_times[proposal.market] > self.current_timestamp: + to_cancel = False + elif cur_orders and self.is_within_tolerance(cur_orders, proposal): + to_cancel = False + if to_cancel: + for order in cur_orders: + self.cancel_order(self._market_infos[proposal.market], order.client_order_id) + # To place new order on the next tick + self._refresh_times[order.trading_pair] = self.current_timestamp + 0.1 def execute_orders_proposal(self, proposals: List[Proposal]): for proposal in proposals: From 418bffc93ed1ea32205669e220a7e1697ce76a66 Mon Sep 17 00:00:00 2001 From: Nullably <37262506+Nullably@users.noreply.github.com> Date: Wed, 24 Feb 2021 14:22:27 +0800 Subject: [PATCH 44/73] (feat) refactor cancel and sort status output --- .../liquidity_mining/liquidity_mining.py | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/hummingbot/strategy/liquidity_mining/liquidity_mining.py b/hummingbot/strategy/liquidity_mining/liquidity_mining.py index c681ce299e..0503dd99d4 100644 --- a/hummingbot/strategy/liquidity_mining/liquidity_mining.py +++ b/hummingbot/strategy/liquidity_mining/liquidity_mining.py @@ -142,8 +142,9 @@ async def active_orders_df(self) -> pd.DataFrame: float(size_q), age_txt ]) - - return pd.DataFrame(data=data, columns=columns) + df = pd.DataFrame(data=data, columns=columns) + df.sort_values(by=["Market", "Side"], inplace=True) + return df def budget_status_df(self) -> pd.DataFrame: data = [] @@ -165,7 +166,9 @@ def budget_status_df(self) -> pd.DataFrame: float(quote_bal), f"{base_pct:.0%} / {quote_pct:.0%}" ]) - return pd.DataFrame(data=data, columns=columns).replace(np.nan, '', regex=True) + df = pd.DataFrame(data=data, columns=columns).replace(np.nan, '', regex=True) + df.sort_values(by=["Market"], inplace=True) + return df def market_status_df(self) -> pd.DataFrame: data = [] @@ -183,7 +186,9 @@ def market_status_df(self) -> pd.DataFrame: f"{best_ask_pct:.2%}", "" if self._volatility[market].is_nan() else f"{self._volatility[market]:.2%}", ]) - return pd.DataFrame(data=data, columns=columns).replace(np.nan, '', regex=True) + df = pd.DataFrame(data=data, columns=columns).replace(np.nan, '', regex=True) + df.sort_values(by=["Market"], inplace=True) + return df async def miner_status_df(self) -> pd.DataFrame: data = [] @@ -200,7 +205,9 @@ async def miner_status_df(self) -> pd.DataFrame: f"{campaign.apy:.2%}", f"{campaign.spread_max:.2%}%" ]) - return pd.DataFrame(data=data, columns=columns).replace(np.nan, '', regex=True) + df = pd.DataFrame(data=data, columns=columns).replace(np.nan, '', regex=True) + df.sort_values(by=["Market"], inplace=True) + return df async def format_status(self) -> str: if not self._ready_to_trade: @@ -327,15 +334,13 @@ def is_within_tolerance(self, cur_orders: List[LimitOrder], proposal: Proposal): def cancel_active_orders(self, proposals: List[Proposal]): for proposal in proposals: - to_cancel = True + to_cancel = False cur_orders = [o for o in self.active_orders if o.trading_pair == proposal.market] if cur_orders and any(self.order_age(o) > self._max_order_age for o in cur_orders): to_cancel = True - else: - if self._refresh_times[proposal.market] > self.current_timestamp: - to_cancel = False - elif cur_orders and self.is_within_tolerance(cur_orders, proposal): - to_cancel = False + elif self._refresh_times[proposal.market] <= self.current_timestamp and \ + cur_orders and not self.is_within_tolerance(cur_orders, proposal): + to_cancel = True if to_cancel: for order in cur_orders: self.cancel_order(self._market_infos[proposal.market], order.client_order_id) From 009dfcab385d13272f61ea6d99cdb836909cb614 Mon Sep 17 00:00:00 2001 From: vic-en Date: Wed, 24 Feb 2021 15:14:20 +0100 Subject: [PATCH 45/73] (fix) factor in leverage when checking balance requirement for derivatives --- .../spot_perpetual_arbitrage/spot_perpetual_arbitrage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py index 3ae784fe62..f3413e2505 100644 --- a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py +++ b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py @@ -264,7 +264,7 @@ def apply_budget_constraint(self, arb_proposal: ArbProposal): spot_token_balance = spot_market.get_available_balance(spot_token) deriv_token_balance = deriv_market.get_available_balance(deriv_token) required_spot_balance = arb_proposal.amount * arb_proposal.spot_side.order_price if arb_proposal.spot_side.is_buy else arb_proposal.amount - required_deriv_balance = arb_proposal.amount * arb_proposal.derivative_side.order_price + required_deriv_balance = (arb_proposal.amount * arb_proposal.derivative_side.order_price) / self._derivative_leverage if spot_token_balance < required_spot_balance: arb_proposal.amount = s_decimal_zero self.logger().info(f"Can't arbitrage, {spot_market.display_name} " From 98e06471ed874a72de58a5bbc764fdc25f858bc6 Mon Sep 17 00:00:00 2001 From: vic-en Date: Wed, 24 Feb 2021 15:42:58 +0100 Subject: [PATCH 46/73] (feat) add trading pair validators --- .../spot_perpetual_arbitrage_config_map.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py index 8be3cd9653..f20b4403a1 100644 --- a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py +++ b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py @@ -1,5 +1,6 @@ from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.config_validators import ( + validate_market_trading_pair, validate_connector, validate_derivative, validate_decimal, @@ -18,10 +19,20 @@ def exchange_on_validated(value: str) -> None: required_exchanges.append(value) +def spot_market_validator(value: str) -> None: + exchange = spot_perpetual_arbitrage_config_map["spot_connector"].value + return validate_market_trading_pair(exchange, value) + + def spot_market_on_validated(value: str) -> None: requried_connector_trading_pairs[spot_perpetual_arbitrage_config_map["spot_connector"].value] = [value] +def derivative_market_validator(value: str) -> None: + exchange = spot_perpetual_arbitrage_config_map["derivative_connector"].value + return validate_market_trading_pair(exchange, value) + + def derivative_market_on_validated(value: str) -> None: requried_connector_trading_pairs[spot_perpetual_arbitrage_config_map["derivative_connector"].value] = [value] @@ -61,6 +72,7 @@ def order_amount_prompt() -> str: key="spot_market", prompt=spot_market_prompt, prompt_on_new=True, + validator=spot_market_validator, on_validated=spot_market_on_validated), "derivative_connector": ConfigVar( key="derivative_connector", @@ -72,6 +84,7 @@ def order_amount_prompt() -> str: key="derivative_market", prompt=derivative_market_prompt, prompt_on_new=True, + validator=derivative_market_validator, on_validated=derivative_market_on_validated), "order_amount": ConfigVar( key="order_amount", From 52273fcee5f6ed480e9b0eb1953cdbadc47927c1 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Thu, 25 Feb 2021 02:31:38 +0800 Subject: [PATCH 47/73] (fix) fix issue where orders not meeting trading rules are still being tracked --- .../exchange/probit/probit_exchange.py | 55 ++++++++++--------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/hummingbot/connector/exchange/probit/probit_exchange.py b/hummingbot/connector/exchange/probit/probit_exchange.py index a78d3d3f76..ebeb4105b7 100644 --- a/hummingbot/connector/exchange/probit/probit_exchange.py +++ b/hummingbot/connector/exchange/probit/probit_exchange.py @@ -463,34 +463,35 @@ async def _create_order(self, amount = self.quantize_order_amount(trading_pair, amount) price = self.quantize_order_price(trading_pair, price) - if amount < trading_rule.min_order_size: - raise ValueError(f"{trade_type.name} order amount {amount} is lower than the minimum order size " - f"{trading_rule.min_order_size}.") - - order_value: Decimal = amount * price - if order_value < trading_rule.min_order_value: - raise ValueError(f"{trade_type.name} order value {order_value} is lower than the minimum order value " - f"{trading_rule.min_order_value}") - - body_params = { - "market_id": trading_pair, - "type": "limit", # ProBit Order Types ["limit", "market"} - "side": trade_type.name.lower(), # ProBit Order Sides ["buy", "sell"] - "time_in_force": "gtc", # gtc = Good-Til-Cancelled - "limit_price": str(price), - "quantity": str(amount), - "client_order_id": order_id - } - - self.start_tracking_order(order_id, - None, - trading_pair, - trade_type, - price, - amount, - order_type - ) try: + if amount < trading_rule.min_order_size: + raise ValueError(f"{trade_type.name} order amount {amount} is lower than the minimum order size " + f"{trading_rule.min_order_size}.") + + order_value: Decimal = amount * price + if order_value < trading_rule.min_order_value: + raise ValueError(f"{trade_type.name} order value {order_value} is lower than the minimum order value " + f"{trading_rule.min_order_value}") + + body_params = { + "market_id": trading_pair, + "type": "limit", # ProBit Order Types ["limit", "market"} + "side": trade_type.name.lower(), # ProBit Order Sides ["buy", "sell"] + "time_in_force": "gtc", # gtc = Good-Til-Cancelled + "limit_price": str(price), + "quantity": str(amount), + "client_order_id": order_id + } + + self.start_tracking_order(order_id, + None, + trading_pair, + trade_type, + price, + amount, + order_type + ) + order_result = await self._api_request( method="POST", path_url=CONSTANTS.NEW_ORDER_URL, From 7fca6387bc474f2811bb48c7469d9aa7c3fe7344 Mon Sep 17 00:00:00 2001 From: vic-en Date: Wed, 24 Feb 2021 20:10:39 +0100 Subject: [PATCH 48/73] (feat) refactor funding payment --- .../perpetual_finance_derivative.py | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py index 48b86ea17f..75980610c1 100644 --- a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py @@ -85,7 +85,8 @@ def __init__(self, self._auto_approve_task = None self._real_time_balance_update = False self._poll_notifier = None - self._funding_payment_span = [1800, 0] + self._funding_payment_span = [120, 120] + self._fundingPayment = {} @property def name(self): @@ -494,16 +495,11 @@ async def _update_balances(self): async def _update_positions(self): position_tasks = [] - funding_payment_tasks = [] for pair in self._trading_pairs: position_tasks.append(self._api_request("post", "perpfi/position", {"pair": convert_to_exchange_trading_pair(pair)})) - funding_payment_tasks.append(self._api_request("get", - "perpfi/funding_payment", - {"pair": convert_to_exchange_trading_pair(pair)})) positions = await safe_gather(*position_tasks, return_exceptions=True) - funding_payments = await safe_gather(*funding_payment_tasks, return_exceptions=True) for trading_pair, position in zip(self._trading_pairs, positions): position = position.get("position", {}) position_side = PositionSide.LONG if position.get("size", 0) > 0 else PositionSide.SHORT @@ -524,17 +520,19 @@ async def _update_positions(self): if (trading_pair + position_side.name) in self._account_positions: del self._account_positions[trading_pair + position_side.name] - for trading_pair, funding_payment in zip(self._trading_pairs, funding_payments): - payment = Decimal(str(funding_payment.payment)) - action = "paid" if payment < 0 else "received" - if payment != Decimal("0"): - self.logger().info(f"Funding payment of {payment} {action} on {trading_pair} market.") - self.trigger_event(MarketEvent.FundingPaymentCompleted, - FundingPaymentCompletedEvent(timestamp=funding_payment.timestamp, - market=self.name, - rate=self._funding_info[trading_pair]["rate"], - symbol=trading_pair, - amount=payment)) + payment = Decimal(str(position.fundingPayment)) + oldPayment = self._fundingPayment.get(trading_pair, 0) + if payment != oldPayment: + self._fundingPayment = oldPayment + action = "paid" if payment < 0 else "received" + if payment != Decimal("0"): + self.logger().info(f"Funding payment of {payment} {action} on {trading_pair} market.") + self.trigger_event(MarketEvent.FundingPaymentCompleted, + FundingPaymentCompletedEvent(timestamp=time.time(), + market=self.name, + rate=self._funding_info[trading_pair]["rate"], + symbol=trading_pair, + amount=payment)) async def _funding_info_polling_loop(self): while True: From e3dd62ccf5b26cdea9c07275b8d9c44555bf6ecd Mon Sep 17 00:00:00 2001 From: Michael Feng Date: Wed, 24 Feb 2021 16:51:10 -0800 Subject: [PATCH 49/73] (feat) adjusted status labels --- hummingbot/strategy/liquidity_mining/liquidity_mining.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hummingbot/strategy/liquidity_mining/liquidity_mining.py b/hummingbot/strategy/liquidity_mining/liquidity_mining.py index 0503dd99d4..e2f25558cb 100644 --- a/hummingbot/strategy/liquidity_mining/liquidity_mining.py +++ b/hummingbot/strategy/liquidity_mining/liquidity_mining.py @@ -123,7 +123,7 @@ def order_age(order: LimitOrder) -> float: return -1. async def active_orders_df(self) -> pd.DataFrame: - size_q_col = f"Size ({self._token})" if self.is_token_a_quote_token() else "Size (Quote)" + size_q_col = f"Amt({self._token})" if self.is_token_a_quote_token() else "Amt(Quote)" columns = ["Market", "Side", "Price", "Spread", "Amount", size_q_col, "Age"] data = [] for order in self.active_orders: @@ -148,7 +148,7 @@ async def active_orders_df(self) -> pd.DataFrame: def budget_status_df(self) -> pd.DataFrame: data = [] - columns = ["Market", f"Budget ({self._token})", "Base Bal", "Quote Bal", "Base / Quote"] + columns = ["Market", f"Budget({self._token})", "Base bal", "Quote bal", "Base/Quote"] for market, market_info in self._market_infos.items(): mid_price = market_info.get_mid_price() base_bal = self._sell_budgets[market] @@ -172,7 +172,7 @@ def budget_status_df(self) -> pd.DataFrame: def market_status_df(self) -> pd.DataFrame: data = [] - columns = ["Market", "Mid Price", "Best Bid %", "Best Ask %", "Volatility"] + columns = ["Market", "Mid price", "Best bid", "Best ask", "Volatility"] for market, market_info in self._market_infos.items(): mid_price = market_info.get_mid_price() best_bid = self._exchange.get_price(market, False) @@ -192,7 +192,7 @@ def market_status_df(self) -> pd.DataFrame: async def miner_status_df(self) -> pd.DataFrame: data = [] - columns = ["Market", "Paid in", "Reward/week", "Curr Liquidity", "APY", "Max Spread"] + columns = ["Market", "Payout", "Reward/wk", "Liquidity", "Yield/yr", "Max spread"] campaigns = await get_campaign_summary(self._exchange.display_name, list(self._market_infos.keys())) for market, campaign in campaigns.items(): reward_usd = await usd_value(campaign.payout_asset, campaign.reward_per_day * Decimal("7")) From f80aa356483c97274ff28e02ce34a8620171492f Mon Sep 17 00:00:00 2001 From: vic-en Date: Thu, 25 Feb 2021 13:16:04 +0100 Subject: [PATCH 50/73] (feat) fix incorrect key for spot connector in config map --- .../spot_perpetual_arbitrage_config_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py index f20b4403a1..ae5083d55b 100644 --- a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py +++ b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py @@ -63,7 +63,7 @@ def order_amount_prompt() -> str: prompt="", default="spot_perpetual_arbitrage"), "spot_connector": ConfigVar( - key="connector_1", + key="spot_connector", prompt="Enter a spot connector (Exchange/AMM) >>> ", prompt_on_new=True, validator=validate_connector, From 2fbc267c9386a1df0c8bcc92855eff99267a713e Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Thu, 25 Feb 2021 20:41:52 +0800 Subject: [PATCH 51/73] (fix) fix partially filled orders[WIP] --- hummingbot/connector/exchange/probit/probit_exchange.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hummingbot/connector/exchange/probit/probit_exchange.py b/hummingbot/connector/exchange/probit/probit_exchange.py index ebeb4105b7..52217a7bc9 100644 --- a/hummingbot/connector/exchange/probit/probit_exchange.py +++ b/hummingbot/connector/exchange/probit/probit_exchange.py @@ -811,7 +811,8 @@ def _process_trade_message(self, order_msg: Dict[str, Any]): tracked_order.executed_amount_base, tracked_order.executed_amount_quote, tracked_order.fee_paid, - tracked_order.order_type)) + tracked_order.order_type, + tracked_order.exchange_order_id)) self.stop_tracking_order(tracked_order.client_order_id) async def get_open_orders(self) -> List[OpenOrder]: From 280da0487b769a8b78ca0e9181df96ed74422c48 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Fri, 26 Feb 2021 10:40:38 +0800 Subject: [PATCH 52/73] (fix) fix _update_with_trade_status() and update process_order_message() --- .../exchange/probit/probit_exchange.py | 24 ++++++++++--------- .../exchange/probit/probit_in_flight_order.py | 11 ++++----- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/hummingbot/connector/exchange/probit/probit_exchange.py b/hummingbot/connector/exchange/probit/probit_exchange.py index 52217a7bc9..6ea763ef60 100644 --- a/hummingbot/connector/exchange/probit/probit_exchange.py +++ b/hummingbot/connector/exchange/probit/probit_exchange.py @@ -716,7 +716,7 @@ async def _update_order_status(self): if isinstance(order_update, Exception): raise order_update if "data" not in order_update: - self.logger().info(f"_update_order_status data not in resp: {order_update}") + self.logger().info(f"Unexpected response from GET /order. 'data' field not in resp: {order_update}") continue for order in order_update["data"]: @@ -742,16 +742,18 @@ def _process_order_message(self, order_msg: Dict[str, Any]): client_order_id)) tracked_order.cancelled_event.set() self.stop_tracking_order(client_order_id) - elif tracked_order.is_failure: - self.logger().info(f"The market order {client_order_id} has failed according to order status API. " - f"Order Message: {order_msg}") - self.trigger_event(MarketEvent.OrderFailure, - MarketOrderFailureEvent( - self.current_timestamp, - client_order_id, - tracked_order.order_type - )) - self.stop_tracking_order(client_order_id) + + # ProBit does not have a 'fail' order status + # elif tracked_order.is_failure: + # self.logger().info(f"The market order {client_order_id} has failed according to order status API. " + # f"Order Message: {order_msg}") + # self.trigger_event(MarketEvent.OrderFailure, + # MarketOrderFailureEvent( + # self.current_timestamp, + # client_order_id, + # tracked_order.order_type + # )) + # self.stop_tracking_order(client_order_id) def _process_trade_message(self, order_msg: Dict[str, Any]): """ diff --git a/hummingbot/connector/exchange/probit/probit_in_flight_order.py b/hummingbot/connector/exchange/probit/probit_in_flight_order.py index fbfddc36fc..daafadca3d 100644 --- a/hummingbot/connector/exchange/probit/probit_in_flight_order.py +++ b/hummingbot/connector/exchange/probit/probit_in_flight_order.py @@ -45,8 +45,8 @@ def is_done(self) -> bool: @property def is_failure(self) -> bool: - # TODO: Determine Order Status Definitions for failed orders - return self.last_state in {"REJECTED"} + # TODO: ProBit does not have a 'fail' order status. + return NotImplementedError @property def is_cancelled(self) -> bool: @@ -84,10 +84,9 @@ def update_with_trade_update(self, trade_update: Dict[str, Any]) -> bool: if str(trade_update["order_id"]) != self.exchange_order_id or trade_id in self.trade_id_set: return False self.trade_id_set.add(trade_id) - self.executed_amount_base += Decimal(str(trade_update["quantity"])) - self.fee_paid += Decimal(str(trade_update["fee_amount"])) - self.executed_amount_quote += (Decimal(str(trade_update["price"])) * - Decimal(str(trade_update["quantity"]))) + self.executed_amount_base = Decimal(str(trade_update["quantity"])) + self.fee_paid = Decimal(str(trade_update["fee_amount"])) + self.executed_amount_quote = Decimal(str(trade_update["cost"])) if not self.fee_asset: self.fee_asset = trade_update["fee_currency_id"] return True From 94ebf5186397bbf318eaccd6f80ba5b20ff569fe Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Fri, 26 Feb 2021 17:53:20 +0800 Subject: [PATCH 53/73] (fix) outstanding issue with handling of partially filled orders --- .../connector/exchange/probit/probit_exchange.py | 12 ++++++++---- .../exchange/probit/probit_in_flight_order.py | 6 +++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/hummingbot/connector/exchange/probit/probit_exchange.py b/hummingbot/connector/exchange/probit/probit_exchange.py index 6ea763ef60..f4ad5a303b 100644 --- a/hummingbot/connector/exchange/probit/probit_exchange.py +++ b/hummingbot/connector/exchange/probit/probit_exchange.py @@ -377,7 +377,9 @@ async def _api_request(self, raise IOError(f"Error parsing data from {path_url}. Error: {str(e)}") if response.status != 200: raise IOError(f"Error fetching data from {path_url}. HTTP status is {response.status}. " - f"Message: {parsed_response}") + f"Message: {parsed_response} " + f"Params: {params} " + f"Data: {data}") return parsed_response @@ -734,7 +736,9 @@ def _process_order_message(self, order_msg: Dict[str, Any]): # Update order execution status tracked_order.last_state = order_msg["status"] - if tracked_order.is_cancelled: + + # NOTE: In ProBit partially-filled orders will retain "filled" status when canceled. + if tracked_order.is_cancelled or Decimal(str(order_msg["cancelled_quantity"])) > Decimal("0"): self.logger().info(f"Successfully cancelled order {client_order_id}.") self.trigger_event(MarketEvent.OrderCancelled, OrderCancelledEvent( @@ -743,7 +747,7 @@ def _process_order_message(self, order_msg: Dict[str, Any]): tracked_order.cancelled_event.set() self.stop_tracking_order(client_order_id) - # ProBit does not have a 'fail' order status + # NOTE: ProBit does not have a 'fail' order status # elif tracked_order.is_failure: # self.logger().info(f"The market order {client_order_id} has failed according to order status API. " # f"Order Message: {order_msg}") @@ -957,7 +961,7 @@ async def _user_stream_event_listener(self): for asset, balance_details in event_message["data"].items(): self._account_balances[asset] = Decimal(str(balance_details["total"])) self._account_available_balances[asset] = Decimal(str(balance_details["available"])) - elif channel in ["open_order", "order_history"]: + elif channel in ["open_order"]: for order_update in event_message["data"]: self._process_order_message(order_update) elif channel == "trade_history": diff --git a/hummingbot/connector/exchange/probit/probit_in_flight_order.py b/hummingbot/connector/exchange/probit/probit_in_flight_order.py index daafadca3d..ab6fdcc0c7 100644 --- a/hummingbot/connector/exchange/probit/probit_in_flight_order.py +++ b/hummingbot/connector/exchange/probit/probit_in_flight_order.py @@ -84,9 +84,9 @@ def update_with_trade_update(self, trade_update: Dict[str, Any]) -> bool: if str(trade_update["order_id"]) != self.exchange_order_id or trade_id in self.trade_id_set: return False self.trade_id_set.add(trade_id) - self.executed_amount_base = Decimal(str(trade_update["quantity"])) - self.fee_paid = Decimal(str(trade_update["fee_amount"])) - self.executed_amount_quote = Decimal(str(trade_update["cost"])) + self.executed_amount_base += Decimal(str(trade_update["quantity"])) + self.fee_paid += Decimal(str(trade_update["fee_amount"])) + self.executed_amount_quote += Decimal(str(trade_update["cost"])) if not self.fee_asset: self.fee_asset = trade_update["fee_currency_id"] return True From 9188c2729be31223df5a5a7ff13983d310da5ffc Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Fri, 26 Feb 2021 19:23:46 +0800 Subject: [PATCH 54/73] (refactor) include global and korea domains to ProBit connector --- .../probit_api_order_book_data_source.py | 23 ++++++++++--------- .../probit_api_user_stream_data_source.py | 17 +++++++++++--- .../connector/exchange/probit/probit_auth.py | 6 +++-- .../exchange/probit/probit_constants.py | 6 ++--- .../exchange/probit/probit_exchange.py | 16 +++++++++---- .../probit/probit_order_book_tracker.py | 12 ++++++---- .../probit/probit_user_stream_tracker.py | 15 ++++++++---- .../connector/exchange/probit/probit_utils.py | 23 ++++++++++++++++++- .../templates/conf_fee_overrides_TEMPLATE.yml | 6 +++++ hummingbot/templates/conf_global_TEMPLATE.yml | 3 +++ 10 files changed, 94 insertions(+), 33 deletions(-) diff --git a/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py b/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py index 3f04d83710..4e6a8550a4 100644 --- a/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py @@ -7,7 +7,7 @@ import ujson import websockets -import hummingbot.connector.exchange.probit.probit_constants as constants +import hummingbot.connector.exchange.probit.probit_constants as CONSTANTS from typing import ( Any, @@ -38,16 +38,17 @@ def logger(cls) -> HummingbotLogger: cls._logger = logging.getLogger(__name__) return cls._logger - def __init__(self, trading_pairs: List[str] = None): + def __init__(self, trading_pairs: List[str] = None, domain: str = "com"): super().__init__(trading_pairs) + self._domain = domain self._trading_pairs: List[str] = trading_pairs self._snapshot_msg: Dict[str, any] = {} @classmethod - async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, float]: + async def get_last_traded_prices(cls, trading_pairs: List[str], domain: str = "com") -> Dict[str, float]: result = {} async with aiohttp.ClientSession() as client: - async with client.get(f"{constants.TICKER_URL}") as response: + async with client.get(f"{CONSTANTS.TICKER_URL.format(domain)}") as response: if response.status == 200: resp_json = await response.json() if "data" in resp_json: @@ -58,25 +59,25 @@ async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, flo return result @staticmethod - async def fetch_trading_pairs() -> List[str]: + async def fetch_trading_pairs(domain: str = "com") -> List[str]: async with aiohttp.ClientSession() as client: - async with client.get(f"{constants.MARKETS_URL}") as response: + async with client.get(f"{CONSTANTS.MARKETS_URL.format(domain)}") as response: if response.status == 200: resp_json: Dict[str, Any] = await response.json() return [market["id"] for market in resp_json["data"]] return [] @staticmethod - async def get_order_book_data(trading_pair: str) -> Dict[str, any]: + async def get_order_book_data(trading_pair: str, domain: str = "com") -> Dict[str, any]: """ Get whole orderbook """ async with aiohttp.ClientSession() as client: - async with client.get(url=f"{constants.ORDER_BOOK_URL}", + async with client.get(url=f"{CONSTANTS.ORDER_BOOK_URL.format(domain)}", params={"market_id": trading_pair}) as response: if response.status != 200: raise IOError( - f"Error fetching OrderBook for {trading_pair} at {constants.ORDER_BOOK_PATH_URL}. " + f"Error fetching OrderBook for {trading_pair} at {CONSTANTS.ORDER_BOOK_PATH_URL.format(domain)}. " f"HTTP {response.status}. Response: {await response.json()}" ) return await response.json() @@ -128,7 +129,7 @@ async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asynci """ while True: try: - async with websockets.connect(uri=constants.WSS_URL) as ws: + async with websockets.connect(uri=CONSTANTS.WSS_URL.format(self._domain)) as ws: ws: websockets.WebSocketClientProtocol = ws for trading_pair in self._trading_pairs: params: Dict[str, Any] = { @@ -169,7 +170,7 @@ async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, outp """ while True: try: - async with websockets.connect(uri=constants.WSS_URL) as ws: + async with websockets.connect(uri=CONSTANTS.WSS_URL.format(self._domain)) as ws: ws: websockets.WebSocketClientProtocol = ws for trading_pair in self._trading_pairs: params: Dict[str, Any] = { diff --git a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py index d88025528b..21b980ae2d 100644 --- a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py @@ -34,7 +34,11 @@ def logger(cls) -> HummingbotLogger: cls._logger = logging.getLogger(__name__) return cls._logger - def __init__(self, probit_auth: ProbitAuth, trading_pairs: Optional[List[str]] = []): + def __init__(self, + probit_auth: ProbitAuth, + trading_pairs: Optional[List[str]] = [], + domain: str = "com"): + self._domain: str = domain self._websocket_client: websockets.WebSocketClientProtocol = None self._probit_auth: ProbitAuth = probit_auth self._trading_pairs = trading_pairs @@ -42,6 +46,13 @@ def __init__(self, probit_auth: ProbitAuth, trading_pairs: Optional[List[str]] = self._last_recv_time: float = 0 super().__init__() + @property + def exchange_name(self) -> str: + if self._domain == "com": + return CONSTANTS.EXCHANGE_NAME + else: + return f"{CONSTANTS.EXCHANGE_NAME}_{self._domain}" + @property def last_recv_time(self) -> float: return self._last_recv_time @@ -52,7 +63,7 @@ async def _init_websocket_connection(self) -> websockets.WebSocketClientProtocol """ try: if self._websocket_client is None: - self._websocket_client = await websockets.connect(CONSTANTS.WSS_URL) + self._websocket_client = await websockets.connect(CONSTANTS.WSS_URL.format(self._domain)) return self._websocket_client except Exception: self.logger().network("Unexpected error occured with ProBit WebSocket Connection") @@ -97,7 +108,7 @@ async def _subscribe_to_channels(self, ws: websockets.WebSocketClientProtocol): except asyncio.CancelledError: raise except Exception: - self.logger().error(f"Error occured subscribing to {CONSTANTS.EXCHANGE_NAME} private channels. ", + self.logger().error(f"Error occured subscribing to {self.exchange_name} private channels. ", exc_info=True) async def _inner_messages(self, diff --git a/hummingbot/connector/exchange/probit/probit_auth.py b/hummingbot/connector/exchange/probit/probit_auth.py index 5401f14c94..b609a09c22 100644 --- a/hummingbot/connector/exchange/probit/probit_auth.py +++ b/hummingbot/connector/exchange/probit/probit_auth.py @@ -15,9 +15,11 @@ class ProbitAuth(): Auth class required by ProBit API Learn more at https://docs-en.probit.com/docs/authorization-1 """ - def __init__(self, api_key: str, secret_key: str): + def __init__(self, api_key: str, secret_key: str, domain: str = "com"): self.api_key: str = api_key self.secret_key: str = secret_key + + self._domain = domain self._oauth_token: str = None self._oauth_token_expiration_time: int = -1 @@ -52,7 +54,7 @@ async def get_auth_headers(self, http_client: aiohttp.ClientSession = aiohttp.Cl body = ujson.dumps({ "grant_type": "client_credentials" }) - resp = await http_client.post(url=CONSTANTS.TOKEN_URL, + resp = await http_client.post(url=CONSTANTS.TOKEN_URL.format(self._domain), headers=headers, data=body) token_resp = await resp.json() diff --git a/hummingbot/connector/exchange/probit/probit_constants.py b/hummingbot/connector/exchange/probit/probit_constants.py index 6583e26a5b..d933e7f91f 100644 --- a/hummingbot/connector/exchange/probit/probit_constants.py +++ b/hummingbot/connector/exchange/probit/probit_constants.py @@ -2,8 +2,8 @@ EXCHANGE_NAME = "probit" -REST_URL = "https://api.probit.com/api/exchange/" -WSS_URL = "wss://api.probit.com/api/exchange/v1/ws" +REST_URL = "https://api.probit.{}/api/exchange/" +WSS_URL = "wss://api.probit.{}/api/exchange/v1/ws" REST_API_VERSON = "v1" @@ -12,7 +12,7 @@ TICKER_URL = f"{REST_URL+REST_API_VERSON}/ticker" MARKETS_URL = f"{REST_URL+REST_API_VERSON}/market" ORDER_BOOK_URL = f"{REST_URL+REST_API_VERSON}/order_book" -TOKEN_URL = "https://accounts.probit.com/token" +TOKEN_URL = "https://accounts.probit.{}/token" # REST API Private Endpoints NEW_ORDER_URL = f"{REST_URL+REST_API_VERSON}/new_order" diff --git a/hummingbot/connector/exchange/probit/probit_exchange.py b/hummingbot/connector/exchange/probit/probit_exchange.py index f4ad5a303b..ae3dd1274e 100644 --- a/hummingbot/connector/exchange/probit/probit_exchange.py +++ b/hummingbot/connector/exchange/probit/probit_exchange.py @@ -71,7 +71,8 @@ def __init__(self, probit_api_key: str, probit_secret_key: str, trading_pairs: Optional[List[str]] = None, - trading_required: bool = True + trading_required: bool = True, + domain="com" ): """ :param probit_api_key: The API key to connect to private ProBit APIs. @@ -79,12 +80,13 @@ def __init__(self, :param trading_pairs: The market trading pairs which to track order book data. :param trading_required: Whether actual trading is needed. """ + self._domain = domain super().__init__() self._trading_required = trading_required self._trading_pairs = trading_pairs - self._probit_auth = ProbitAuth(probit_api_key, probit_secret_key) - self._order_book_tracker = ProbitOrderBookTracker(trading_pairs=trading_pairs) - self._user_stream_tracker = ProbitUserStreamTracker(self._probit_auth, trading_pairs) + self._probit_auth = ProbitAuth(probit_api_key, probit_secret_key, domain=domain) + self._order_book_tracker = ProbitOrderBookTracker(trading_pairs=trading_pairs, domain=domain) + self._user_stream_tracker = ProbitUserStreamTracker(self._probit_auth, trading_pairs, domain=domain) self._ev_loop = asyncio.get_event_loop() self._shared_client = None self._poll_notifier = asyncio.Event() @@ -101,7 +103,10 @@ def __init__(self, @property def name(self) -> str: - return CONSTANTS.EXCHANGE_NAME + if self._domain == "com": + return CONSTANTS.EXCHANGE_NAME + else: + return f"{CONSTANTS.EXCHANGE_NAME}_{self._domain}" @property def order_books(self) -> Dict[str, OrderBook]: @@ -357,6 +362,7 @@ async def _api_request(self, signature to the request. :returns A response in json format. """ + path_url = path_url.format(self._domain) client = await self._http_client() if is_auth_required: diff --git a/hummingbot/connector/exchange/probit/probit_order_book_tracker.py b/hummingbot/connector/exchange/probit/probit_order_book_tracker.py index 8c0c1d0fad..e7ac692ba1 100644 --- a/hummingbot/connector/exchange/probit/probit_order_book_tracker.py +++ b/hummingbot/connector/exchange/probit/probit_order_book_tracker.py @@ -2,7 +2,7 @@ import asyncio import bisect import logging -import hummingbot.connector.exchange.probit.probit_constants as constants +import hummingbot.connector.exchange.probit.probit_constants as CONSTANTS import time from collections import defaultdict, deque @@ -25,9 +25,10 @@ def logger(cls) -> HummingbotLogger: cls._logger = logging.getLogger(__name__) return cls._logger - def __init__(self, trading_pairs: Optional[List[str]] = None,): - super().__init__(ProbitAPIOrderBookDataSource(trading_pairs), trading_pairs) + def __init__(self, trading_pairs: Optional[List[str]] = None, domain: str = "com"): + super().__init__(ProbitAPIOrderBookDataSource(trading_pairs, domain), trading_pairs) + self._domain = domain self._ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() self._order_book_snapshot_stream: asyncio.Queue = asyncio.Queue() self._order_book_diff_stream: asyncio.Queue = asyncio.Queue() @@ -45,7 +46,10 @@ def exchange_name(self) -> str: """ Name of the current exchange """ - return constants.EXCHANGE_NAME + if self._domain == "com": + return CONSTANTS.EXCHANGE_NAME + else: + return f"{CONSTANTS.EXCHANGE_NAME}_{self._domain}" async def _track_single_book(self, trading_pair: str): """ diff --git a/hummingbot/connector/exchange/probit/probit_user_stream_tracker.py b/hummingbot/connector/exchange/probit/probit_user_stream_tracker.py index d1bcb1cbb8..b057cda3ef 100644 --- a/hummingbot/connector/exchange/probit/probit_user_stream_tracker.py +++ b/hummingbot/connector/exchange/probit/probit_user_stream_tracker.py @@ -3,13 +3,14 @@ import asyncio import logging +import hummingbot.connector.exchange.probit.probit_constants as CONSTANTS + from typing import ( Optional, List, ) from hummingbot.connector.exchange.probit.probit_auth import ProbitAuth -from hummingbot.connector.exchange.probit.probit_constants import EXCHANGE_NAME from hummingbot.connector.exchange.probit.probit_api_user_stream_data_source import \ ProbitAPIUserStreamDataSource from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource @@ -32,8 +33,10 @@ def logger(cls) -> HummingbotLogger: def __init__(self, probit_auth: Optional[ProbitAuth] = None, - trading_pairs: Optional[List[str]] = []): + trading_pairs: Optional[List[str]] = [], + domain: str = "com"): super().__init__() + self._domain: str = domain self._probit_auth: ProbitAuth = probit_auth self._trading_pairs: List[str] = trading_pairs self._ev_loop: asyncio.events.AbstractEventLoop = asyncio.get_event_loop() @@ -50,7 +53,8 @@ def data_source(self) -> UserStreamTrackerDataSource: if not self._data_source: self._data_source = ProbitAPIUserStreamDataSource( probit_auth=self._probit_auth, - trading_pairs=self._trading_pairs + trading_pairs=self._trading_pairs, + domain=self._domain ) return self._data_source @@ -60,7 +64,10 @@ def exchange_name(self) -> str: *required Name of the current exchange """ - return EXCHANGE_NAME + if self._domain == "com": + return CONSTANTS.EXCHANGE_NAME + else: + return f"{CONSTANTS.EXCHANGE_NAME}_{self._domain}" async def start(self): """ diff --git a/hummingbot/connector/exchange/probit/probit_utils.py b/hummingbot/connector/exchange/probit/probit_utils.py index 5920ef47d7..2631e9ec41 100644 --- a/hummingbot/connector/exchange/probit/probit_utils.py +++ b/hummingbot/connector/exchange/probit/probit_utils.py @@ -75,7 +75,7 @@ def convert_diff_message_to_order_book_row(message: OrderBookMessage) -> Tuple[L KEYS = { "probit_api_key": ConfigVar(key="probit_api_key", - prompt="Enter your ProBit API key >>> ", + prompt="Enter your ProBit Client ID >>> ", required_if=using_exchange("probit"), is_secure=True, is_connect_key=True), @@ -86,3 +86,24 @@ def convert_diff_message_to_order_book_row(message: OrderBookMessage) -> Tuple[L is_secure=True, is_connect_key=True), } + +OTHER_DOMAINS = ["probit_kr"] +OTHER_DOMAINS_PARAMETER = {"probit_kr": "kr"} +OTHER_DOMAINS_EXAMPLE_PAIR = {"probit_kr": "BTC-USDT"} +OTHER_DOMAINS_DEFAULT_FEES = {"probit_kr": [0.2, 0.2]} +OTHER_DOMAINS_KEYS = { + "probit_kr": { + "probit_kr_api_key": + ConfigVar(key="probit_kr_api_key", + prompt="Enter your ProBit KR Client ID >>> ", + required_if=using_exchange("probit_kr"), + is_secure=True, + is_connect_key=True), + "probit_kr_secret_key": + ConfigVar(key="probit_kr_secret_key", + prompt="Enter your ProBit KR secret key >>> ", + required_if=using_exchange("probit_kr"), + is_secure=True, + is_connect_key=True), + } +} diff --git a/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml b/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml index 506eb80b09..dd3f6ca402 100644 --- a/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml +++ b/hummingbot/templates/conf_fee_overrides_TEMPLATE.yml @@ -70,3 +70,9 @@ balancer_taker_fee_amount: bitmax_maker_fee: bitmax_taker_fee: + +probit_maker_fee: +probit_taker_fee: + +probit_kr_maker_fee: +probit_kr_taker_fee: diff --git a/hummingbot/templates/conf_global_TEMPLATE.yml b/hummingbot/templates/conf_global_TEMPLATE.yml index 0f41f4d3b0..e7afe14b40 100644 --- a/hummingbot/templates/conf_global_TEMPLATE.yml +++ b/hummingbot/templates/conf_global_TEMPLATE.yml @@ -80,6 +80,9 @@ balancer_max_swaps: 4 probit_api_key: null probit_secret_key: null +probit_kr_api_key: null +probit_kr_secret_key: null + # Ethereum wallet address: required for trading on a DEX ethereum_wallet: null ethereum_rpc_url: null From 5dfd53e481bfa928b5c3372a2de6648805408341 Mon Sep 17 00:00:00 2001 From: vic-en Date: Mon, 1 Mar 2021 01:26:32 +0100 Subject: [PATCH 55/73] (fix) made request changes and fixed some bugs related to funding payment completion --- .../binance_perpetual_derivative.py | 4 +-- .../perpetual_finance_derivative.py | 34 +++++++++++-------- .../perpetual_finance_in_flight_order.py | 17 ---------- .../perpetual_finance_utils.py | 4 +-- hummingbot/core/event/events.py | 1 + .../spot_perpetual_arbitrage/arb_proposal.py | 1 - .../spot_perpetual_arbitrage.py | 15 ++++---- 7 files changed, 33 insertions(+), 43 deletions(-) diff --git a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py index 8a612e7890..94a4bbb2e5 100644 --- a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py +++ b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py @@ -947,8 +947,8 @@ async def get_funding_payment(self): self.trigger_event(self.MARKET_FUNDING_PAYMENT_COMPLETED_EVENT_TAG, FundingPaymentCompletedEvent(timestamp=funding_payment["time"], market=self.name, - rate=self._funding_info[trading_pair]["rate"], - symbol=trading_pair, + funding_rate=self._funding_info[trading_pair]["rate"], + trading_pair=trading_pair, amount=payment)) def get_funding_info(self, trading_pair): diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py index 75980610c1..17388e23e9 100644 --- a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py @@ -8,6 +8,7 @@ import ssl import copy from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL +from hummingbot.core.event.events import TradeFee from hummingbot.core.utils import async_ttl_cache from hummingbot.core.network_iterator import NetworkStatus from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather @@ -180,7 +181,7 @@ async def get_order_price(self, trading_pair: str, is_buy: bool, amount: Decimal """ return await self.get_quote_price(trading_pair, is_buy, amount) - def buy(self, trading_pair: str, amount: Decimal, order_type: OrderType, price: Decimal, position_action: PositionAction) -> str: + def buy(self, trading_pair: str, amount: Decimal, order_type: OrderType, price: Decimal, **kwargs) -> str: """ Buys an amount of base token for a given price (or cheaper). :param trading_pair: The market trading pair @@ -190,9 +191,9 @@ def buy(self, trading_pair: str, amount: Decimal, order_type: OrderType, price: :param position_action: Either OPEN or CLOSE position action. :return: A newly created order id (internal). """ - return self.place_order(True, trading_pair, amount, price, position_action) + return self.place_order(True, trading_pair, amount, price, kwargs["position_action"]) - def sell(self, trading_pair: str, amount: Decimal, order_type: OrderType, price: Decimal, position_action: PositionAction) -> str: + def sell(self, trading_pair: str, amount: Decimal, order_type: OrderType, price: Decimal, **kwargs) -> str: """ Sells an amount of base token for a given price (or at a higher price). :param trading_pair: The market trading pair @@ -202,7 +203,7 @@ def sell(self, trading_pair: str, amount: Decimal, order_type: OrderType, price: :param position_action: Either OPEN or CLOSE position action. :return: A newly created order id (internal). """ - return self.place_order(False, trading_pair, amount, price, position_action) + return self.place_order(False, trading_pair, amount, price, kwargs["position_action"]) def place_order(self, is_buy: bool, trading_pair: str, amount: Decimal, price: Decimal, position_action: PositionAction) -> str: """ @@ -242,9 +243,9 @@ async def _create_order(self, api_params = {"pair": convert_to_exchange_trading_pair(trading_pair)} if position_action == PositionAction.OPEN: api_params.update({"side": 0 if trade_type == TradeType.BUY else 1, - "margin": str(amount / self._leverage[trading_pair]), + "margin": self.quantize_order_amount(trading_pair, (amount / self._leverage[trading_pair])), "leverage": self._leverage[trading_pair], - "minBaseAssetAmount": amount}) + "minBaseAssetAmount": Decimal("0")}) else: api_params.update({"minimalQuoteAsset": price * amount}) self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, self._leverage[trading_pair], position_action.name) @@ -337,7 +338,7 @@ async def _update_order_status(self): if update_result["confirmed"] is True: if update_result["receipt"]["status"] == 1: fee = estimate_fee("perpetual_finance", False) - fee.flat_fees = [(tracked_order.fee_asset, Decimal(str(update_result["receipt"]["gasUsed"])))] + fee = TradeFee(fee.percent, [("XDAI", Decimal(str(update_result["receipt"]["gasUsed"])))]) self.trigger_event( MarketEvent.OrderFilled, OrderFilledEvent( @@ -370,7 +371,9 @@ async def _update_order_status(self): tracked_order.fee_asset, tracked_order.executed_amount_base, tracked_order.executed_amount_quote, - float(fee), + float(fee.fee_amount_in_quote(tracked_order.trading_pair, + Decimal(str(tracked_order.price)), + Decimal(str(tracked_order.amount)))), # this ignores the gas fee, which is fine for now tracked_order.order_type)) self.stop_tracking_order(tracked_order.client_order_id) else: @@ -388,10 +391,10 @@ def get_taker_order_type(self): return OrderType.LIMIT def get_order_price_quantum(self, trading_pair: str, price: Decimal) -> Decimal: - return Decimal("1e-15") + return Decimal("1e-6") def get_order_size_quantum(self, trading_pair: str, order_size: Decimal) -> Decimal: - return Decimal("1e-15") + return Decimal("1e-6") @property def ready(self): @@ -454,6 +457,7 @@ async def _status_polling_loop(self): self._poll_notifier = asyncio.Event() await self._poll_notifier.wait() await safe_gather( + self._update_positions(), self._update_balances(), self._update_order_status(), ) @@ -502,7 +506,7 @@ async def _update_positions(self): positions = await safe_gather(*position_tasks, return_exceptions=True) for trading_pair, position in zip(self._trading_pairs, positions): position = position.get("position", {}) - position_side = PositionSide.LONG if position.get("size", 0) > 0 else PositionSide.SHORT + position_side = PositionSide.LONG if Decimal(position.get("size", "0")) > 0 else PositionSide.SHORT unrealized_pnl = Decimal(position.get("pnl")) entry_price = Decimal(position.get("entryPrice")) amount = Decimal(position.get("size")) @@ -520,18 +524,18 @@ async def _update_positions(self): if (trading_pair + position_side.name) in self._account_positions: del self._account_positions[trading_pair + position_side.name] - payment = Decimal(str(position.fundingPayment)) + payment = Decimal(str(position.get("fundingPayment"))) oldPayment = self._fundingPayment.get(trading_pair, 0) if payment != oldPayment: - self._fundingPayment = oldPayment + self._fundingPayment[trading_pair] = oldPayment action = "paid" if payment < 0 else "received" if payment != Decimal("0"): self.logger().info(f"Funding payment of {payment} {action} on {trading_pair} market.") self.trigger_event(MarketEvent.FundingPaymentCompleted, FundingPaymentCompletedEvent(timestamp=time.time(), market=self.name, - rate=self._funding_info[trading_pair]["rate"], - symbol=trading_pair, + funding_rate=self._funding_info[trading_pair]["rate"], + trading_pair=trading_pair, amount=payment)) async def _funding_info_polling_loop(self): diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_in_flight_order.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_in_flight_order.py index 3dbaad3128..ff2df9b255 100644 --- a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_in_flight_order.py +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_in_flight_order.py @@ -31,7 +31,6 @@ def __init__(self, amount, initial_state, ) - self.trade_id_set = set() self.leverage = leverage self.position = position @@ -46,19 +45,3 @@ def is_failure(self) -> bool: @property def is_cancelled(self) -> bool: return self.last_state in {"CANCELED", "EXPIRED"} - - @property - def leverage(self) -> Decimal: - return self.leverage - - @leverage.setter - def leverage(self, leverage) -> Decimal: - self.leverage = leverage - - @property - def position(self) -> Decimal: - return self.position - - @position.setter - def position(self, position) -> Decimal: - self.position = position diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_utils.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_utils.py index 665a0a34e1..d7f1b5f60a 100644 --- a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_utils.py +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_utils.py @@ -6,8 +6,8 @@ DEFAULT_FEES = [0.1, 0.1] USE_ETHEREUM_WALLET = True -FEE_TYPE = "FlatFee" -FEE_TOKEN = "XDAI" +# FEE_TYPE = "FlatFee" +# FEE_TOKEN = "XDAI" USE_ETH_GAS_LOOKUP = False diff --git a/hummingbot/core/event/events.py b/hummingbot/core/event/events.py index c4a02d1e76..055f7d2cac 100644 --- a/hummingbot/core/event/events.py +++ b/hummingbot/core/event/events.py @@ -207,6 +207,7 @@ class OrderExpiredEvent(NamedTuple): @dataclass class FundingPaymentCompletedEvent: timestamp: float + market: str trading_pair: str amount: Decimal funding_rate: Decimal diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/arb_proposal.py b/hummingbot/strategy/spot_perpetual_arbitrage/arb_proposal.py index d6c78cffff..c24a85eec0 100644 --- a/hummingbot/strategy/spot_perpetual_arbitrage/arb_proposal.py +++ b/hummingbot/strategy/spot_perpetual_arbitrage/arb_proposal.py @@ -19,7 +19,6 @@ def __init__(self, """ :param market_info: The market where to submit the order :param is_buy: True if buy order - :param quote_price: The quote price (for an order amount) from the market :param order_price: The price required for order submission, this could differ from the quote price :param amount: The order amount """ diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py index f3413e2505..824280d0f6 100644 --- a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py +++ b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py @@ -169,6 +169,8 @@ async def main(self, timestamp): funding_msg = "Time for funding payment, executing second arbitrage " \ "immediately since we don't intend to maximize funding rate" execute_arb = True + else: + funding_msg = "Funding payment time, not looking for arbitrage opportunity because prices should be converging now!" else: if len(self.deriv_position) > 0: execute_arb = self.ready_for_execution(self.current_proposal, False) @@ -181,8 +183,10 @@ async def main(self, timestamp): self.apply_budget_constraint(self.current_proposal) await self.execute_arb_proposals(self.current_proposal, funding_msg) else: - self.timed_logger(timestamp, self.spread_msg()) - return + if funding_msg: + self.timed_logger(timestamp, funding_msg) + else: + self.timed_logger(timestamp, self.spread_msg()) def timed_logger(self, timestamp, msg): """ @@ -238,7 +242,7 @@ def apply_slippage_buffers(self, arb_proposal: ArbProposal): Updates arb_proposals by adjusting order price for slipper buffer percentage. E.g. if it is a buy order, for an order price of 100 and 1% slipper buffer, the new order price is 101, for a sell order, the new order price is 99. - :param arb_proposals: the arbitrage proposal + :param arb_proposal: the arbitrage proposal """ for arb_side in (arb_proposal.spot_side, arb_proposal.derivative_side): market = arb_side.market_info.market @@ -255,7 +259,7 @@ def apply_budget_constraint(self, arb_proposal: ArbProposal): """ Updates arb_proposals by setting proposal amount to 0 if there is not enough balance to submit order with required order amount. - :param arb_proposals: the arbitrage proposal + :param arb_proposal: the arbitrage proposal """ spot_market = self._spot_market_info.market deriv_market = self._derivative_market_info.market @@ -275,7 +279,6 @@ def apply_budget_constraint(self, arb_proposal: ArbProposal): self.logger().info(f"Can't arbitrage, {deriv_market.display_name} " f"{deriv_token} balance " f"({deriv_token_balance}) is below required order amount ({required_deriv_balance}).") - return async def execute_arb_proposals(self, arb_proposal: ArbProposal, is_funding_msg: str = ""): """ @@ -444,7 +447,7 @@ def did_expire_order(self, expired_event): def did_complete_funding_payment(self, funding_payment_completed_event): # Excute second arbitrage if necessary (even spread hasn't reached min convergence) - if len(self.deriv_position) > 0: + if len(self.deriv_position) > 0 and self.ready_for_new_arb_trades(): self.apply_slippage_buffers(self.current_proposal) self.apply_budget_constraint(self.current_proposal) funding_msg = "Executing second arbitrage after funding payment is received" From a2f4ee5b2b07ed97aeae4ab216e8d6118e41cdd3 Mon Sep 17 00:00:00 2001 From: vic-en Date: Mon, 1 Mar 2021 10:47:23 +0100 Subject: [PATCH 56/73] (fix) fix perp protocol update_position_function --- .../perpetual_finance_derivative.py | 61 ++++++++++--------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py index 17388e23e9..b752a5bf35 100644 --- a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py @@ -506,37 +506,38 @@ async def _update_positions(self): positions = await safe_gather(*position_tasks, return_exceptions=True) for trading_pair, position in zip(self._trading_pairs, positions): position = position.get("position", {}) - position_side = PositionSide.LONG if Decimal(position.get("size", "0")) > 0 else PositionSide.SHORT - unrealized_pnl = Decimal(position.get("pnl")) - entry_price = Decimal(position.get("entryPrice")) amount = Decimal(position.get("size")) - leverage = self._leverage[trading_pair] - if amount != 0: - self._account_positions[trading_pair + position_side.name] = Position( - trading_pair=trading_pair, - position_side=position_side, - unrealized_pnl=unrealized_pnl, - entry_price=entry_price, - amount=amount, - leverage=leverage - ) - else: - if (trading_pair + position_side.name) in self._account_positions: - del self._account_positions[trading_pair + position_side.name] - - payment = Decimal(str(position.get("fundingPayment"))) - oldPayment = self._fundingPayment.get(trading_pair, 0) - if payment != oldPayment: - self._fundingPayment[trading_pair] = oldPayment - action = "paid" if payment < 0 else "received" - if payment != Decimal("0"): - self.logger().info(f"Funding payment of {payment} {action} on {trading_pair} market.") - self.trigger_event(MarketEvent.FundingPaymentCompleted, - FundingPaymentCompletedEvent(timestamp=time.time(), - market=self.name, - funding_rate=self._funding_info[trading_pair]["rate"], - trading_pair=trading_pair, - amount=payment)) + if amount != Decimal("0"): + position_side = PositionSide.LONG if amount > 0 else PositionSide.SHORT + unrealized_pnl = Decimal(position.get("pnl")) + entry_price = Decimal(position.get("entryPrice")) + leverage = self._leverage[trading_pair] + if amount != 0: + self._account_positions[trading_pair + position_side.name] = Position( + trading_pair=trading_pair, + position_side=position_side, + unrealized_pnl=unrealized_pnl, + entry_price=entry_price, + amount=amount, + leverage=leverage + ) + else: + if (trading_pair + position_side.name) in self._account_positions: + del self._account_positions[trading_pair + position_side.name] + + payment = Decimal(str(position.get("fundingPayment"))) + oldPayment = self._fundingPayment.get(trading_pair, 0) + if payment != oldPayment: + self._fundingPayment[trading_pair] = oldPayment + action = "paid" if payment < 0 else "received" + if payment != Decimal("0"): + self.logger().info(f"Funding payment of {payment} {action} on {trading_pair} market.") + self.trigger_event(MarketEvent.FundingPaymentCompleted, + FundingPaymentCompletedEvent(timestamp=time.time(), + market=self.name, + funding_rate=self._funding_info[trading_pair]["rate"], + trading_pair=trading_pair, + amount=payment)) async def _funding_info_polling_loop(self): while True: From f9af9a137a4874731e4007ae7aab3822dd2e7c41 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Mon, 1 Mar 2021 19:04:36 +0800 Subject: [PATCH 57/73] (refactor) include get_ws_auth_payload() into ProbitAuth --- .../exchange/probit/probit_api_user_stream_data_source.py | 6 +----- hummingbot/connector/exchange/probit/probit_auth.py | 7 +++++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py index 21b980ae2d..b0e9f185de 100644 --- a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py @@ -73,11 +73,7 @@ async def _authenticate(self, ws: websockets.WebSocketClientProtocol): Authenticates user to websocket """ try: - await self._probit_auth.get_auth_headers() - auth_payload: Dict[str, Any] = { - "type": "authorization", - "token": self._probit_auth.oauth_token - } + auth_payload: Dict[str, Any] = self._probit_auth.get_ws_auth_payload() await ws.send(ujson.dumps(auth_payload, escape_forward_slashes=False)) auth_resp = await ws.recv() auth_resp: Dict[str, Any] = ujson.loads(auth_resp) diff --git a/hummingbot/connector/exchange/probit/probit_auth.py b/hummingbot/connector/exchange/probit/probit_auth.py index b609a09c22..5616c617cd 100644 --- a/hummingbot/connector/exchange/probit/probit_auth.py +++ b/hummingbot/connector/exchange/probit/probit_auth.py @@ -72,6 +72,13 @@ async def get_auth_headers(self, http_client: aiohttp.ClientSession = aiohttp.Cl return self.generate_auth_dict() + async def get_ws_auth_payload(self) -> Dict[str, Any]: + await self.get_auth_headers() + return { + "type": "authorization", + "token": self._oauth_token + } + def generate_auth_dict(self): """ Generates authentication signature and return it in a dictionary along with other inputs From e93b8d511ef0e06b88a5fe206d926161e39f9f3d Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Mon, 1 Mar 2021 19:06:29 +0800 Subject: [PATCH 58/73] (add) add ProBitAuthUnitTest --- test/connector/exchange/probit/__init__.py | 0 .../exchange/probit/test_probit_auth.py | 62 +++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 test/connector/exchange/probit/__init__.py create mode 100644 test/connector/exchange/probit/test_probit_auth.py diff --git a/test/connector/exchange/probit/__init__.py b/test/connector/exchange/probit/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/connector/exchange/probit/test_probit_auth.py b/test/connector/exchange/probit/test_probit_auth.py new file mode 100644 index 0000000000..4d6ccdab51 --- /dev/null +++ b/test/connector/exchange/probit/test_probit_auth.py @@ -0,0 +1,62 @@ +import aiohttp +import asyncio +import conf +import logging +import sys +import unittest +import ujson +import websockets + +import hummingbot.connector.exchange.probit.probit_constants as CONSTANTS + +from os.path import join, realpath +from typing import ( + Any, + Dict, +) + +from hummingbot.connector.exchange.probit.probit_auth import ProbitAuth +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL + +sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +logging.basicConfig(level=METRICS_LOG_LEVEL) + + +class ProBitAuthUnitTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + api_key = conf.probit_api_key + secret_key = conf.crypto_com_secret_key + cls.auth: ProbitAuth = ProbitAuth(api_key, secret_key) + + async def rest_auth(self) -> Dict[str, Any]: + http_client = aiohttp.ClientSession() + resp = await self.auth.get_auth_headers(http_client) + await http_client.close() + return resp + + async def ws_auth(self) -> Dict[Any, Any]: + ws = await websockets.connect(CONSTANTS.WSS_URL.format("com")) + + auth_payload = await self.auth.get_ws_auth_payload() + + await ws.send(ujson.dumps(auth_payload, escape_forward_slashes=False)) + resp = await ws.recv() + await ws.close() + + return ujson.loads(resp) + + def test_rest_auth(self): + result = self.ev_loop.run_until_complete(self.rest_auth()) + assert not isinstance(result, Exception) + + def test_ws_auth(self): + result = self.ev_loop.run_until_complete(self.ws_auth()) + assert result["result"] == "ok" + + +if __name__ == "__main__": + logging.getLogger("hummingbot.core.event.event_reporter").setLevel(logging.WARNING) + unittest.main() From ae7ddc9b06591e2beeaa2848f2ec52bfd8680ba9 Mon Sep 17 00:00:00 2001 From: vic-en Date: Mon, 1 Mar 2021 14:46:16 +0100 Subject: [PATCH 59/73] (fix) alternate logs when waiting for funding payment --- .../spot_perpetual_arbitrage/spot_perpetual_arbitrage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py index 824280d0f6..852502859e 100644 --- a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py +++ b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py @@ -162,9 +162,9 @@ async def main(self, timestamp): if self._maximize_funding_rate: execute_arb = not self.would_receive_funding_payment(self.deriv_position) if execute_arb: - self.timed_logger(timestamp, "Waiting for funding payment.") - else: funding_msg = "Time for funding payment, executing second arbitrage to prevent paying funding fee" + else: + funding_msg = "Waiting for funding payment." else: funding_msg = "Time for funding payment, executing second arbitrage " \ "immediately since we don't intend to maximize funding rate" From 37997c87d187836698dcdf1efded2081fa4b4c61 Mon Sep 17 00:00:00 2001 From: vic-en Date: Mon, 1 Mar 2021 14:59:10 +0100 Subject: [PATCH 60/73] (fix) add market validators --- hummingbot/strategy/amm_arb/amm_arb_config_map.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/hummingbot/strategy/amm_arb/amm_arb_config_map.py b/hummingbot/strategy/amm_arb/amm_arb_config_map.py index 16e15e90c7..f662855f10 100644 --- a/hummingbot/strategy/amm_arb/amm_arb_config_map.py +++ b/hummingbot/strategy/amm_arb/amm_arb_config_map.py @@ -1,5 +1,6 @@ from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.config_validators import ( + validate_market_trading_pair, validate_connector, validate_decimal, validate_bool @@ -16,10 +17,20 @@ def exchange_on_validated(value: str) -> None: required_exchanges.append(value) +def market_1_validator(value: str) -> None: + exchange = amm_arb_config_map["connector_1"].value + return validate_market_trading_pair(exchange, value) + + def market_1_on_validated(value: str) -> None: requried_connector_trading_pairs[amm_arb_config_map["connector_1"].value] = [value] +def market_2_validator(value: str) -> None: + exchange = amm_arb_config_map["connector_2"].value + return validate_market_trading_pair(exchange, value) + + def market_2_on_validated(value: str) -> None: requried_connector_trading_pairs[amm_arb_config_map["connector_2"].value] = [value] @@ -59,6 +70,7 @@ def order_amount_prompt() -> str: key="market_1", prompt=market_1_prompt, prompt_on_new=True, + validator=market_1_on_validated, on_validated=market_1_on_validated), "connector_2": ConfigVar( key="connector_2", @@ -70,6 +82,7 @@ def order_amount_prompt() -> str: key="market_2", prompt=market_2_prompt, prompt_on_new=True, + validator=market_2_validator, on_validated=market_2_on_validated), "order_amount": ConfigVar( key="order_amount", From 28f4dd1e8c92548a7e448e138b4edc81f31ccd0e Mon Sep 17 00:00:00 2001 From: vic-en Date: Mon, 1 Mar 2021 22:09:03 +0100 Subject: [PATCH 61/73] (fix) fixed some errors related to perpetual protocol --- ...tual_finance_api_order_book_data_source.py | 10 +++++++ .../perpetual_finance_derivative.py | 28 +++++++++---------- .../spot_perpetual_arbitrage.py | 16 +++++++---- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_api_order_book_data_source.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_api_order_book_data_source.py index 4abc57b74c..2df75a53be 100644 --- a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_api_order_book_data_source.py +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_api_order_book_data_source.py @@ -2,6 +2,7 @@ from typing import List import json import ssl +from typing import Dict from hummingbot.client.config.global_config_map import global_config_map from hummingbot.connector.derivative.perpetual_finance.perpetual_finance_utils import convert_from_exchange_trading_pair @@ -41,3 +42,12 @@ async def fetch_trading_pairs() -> List[str]: for pair in pairs: trading_pairs.append(convert_from_exchange_trading_pair(pair)) return trading_pairs + + @classmethod + async def get_last_traded_prices(cls, trading_pairs: List[str]) -> Dict[str, float]: + """ + This function doesn't really need to return a value. + It is only currently used for performance calculation which will in turn use the last price of the last trades + if None is returned. + """ + pass diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py index b752a5bf35..e8ea75a72a 100644 --- a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py @@ -243,11 +243,12 @@ async def _create_order(self, api_params = {"pair": convert_to_exchange_trading_pair(trading_pair)} if position_action == PositionAction.OPEN: api_params.update({"side": 0 if trade_type == TradeType.BUY else 1, - "margin": self.quantize_order_amount(trading_pair, (amount / self._leverage[trading_pair])), + "margin": self.quantize_order_amount(trading_pair, (amount / self._leverage[trading_pair] * price)), "leverage": self._leverage[trading_pair], "minBaseAssetAmount": Decimal("0")}) else: - api_params.update({"minimalQuoteAsset": price * amount}) + # api_params.update({"minimalQuoteAsset": price * amount}) + api_params.update({"minimalQuoteAsset": Decimal("0")}) self.start_tracking_order(order_id, None, trading_pair, trade_type, price, amount, self._leverage[trading_pair], position_action.name) try: order_result = await self._api_request("post", f"perpfi/{position_action.name.lower()}", api_params) @@ -512,18 +513,17 @@ async def _update_positions(self): unrealized_pnl = Decimal(position.get("pnl")) entry_price = Decimal(position.get("entryPrice")) leverage = self._leverage[trading_pair] - if amount != 0: - self._account_positions[trading_pair + position_side.name] = Position( - trading_pair=trading_pair, - position_side=position_side, - unrealized_pnl=unrealized_pnl, - entry_price=entry_price, - amount=amount, - leverage=leverage - ) - else: - if (trading_pair + position_side.name) in self._account_positions: - del self._account_positions[trading_pair + position_side.name] + self._account_positions[trading_pair] = Position( + trading_pair=trading_pair, + position_side=position_side, + unrealized_pnl=unrealized_pnl, + entry_price=entry_price, + amount=amount, + leverage=leverage + ) + else: + if trading_pair in self._account_positions: + del self._account_positions[trading_pair] payment = Decimal(str(position.get("fundingPayment"))) oldPayment = self._fundingPayment.get(trading_pair, 0) diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py index 852502859e..6838e5d20e 100644 --- a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py +++ b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py @@ -369,10 +369,8 @@ def spread_msg(self): def active_positions_df(self) -> pd.DataFrame: columns = ["Symbol", "Type", "Entry Price", "Amount", "Leverage", "Unrealized PnL"] data = [] - market, trading_pair = self._derivative_market_info.market, self._derivative_market_info.trading_pair for idx in self.deriv_position: - is_buy = True if idx.amount > 0 else False - unrealized_profit = ((market.get_price(trading_pair, is_buy) - idx.entry_price) * idx.amount) + unrealized_profit = ((self.current_proposal.derivative_side.order_price - idx.entry_price) * idx.amount) data.append([ idx.trading_pair, idx.position_side.name, @@ -418,8 +416,11 @@ async def format_status(self) -> str: lines.extend(["", " Assets:"] + [" " + line for line in str(assets_df).split("\n")]) - lines.extend(["", " Spread details:"] + [" " + self.spread_msg()] + - self.short_proposal_msg()) + try: + lines.extend(["", " Spread details:"] + [" " + self.spread_msg()] + + self.short_proposal_msg()) + except Exception: + pass warning_lines = self.network_warning([self._spot_market_info]) warning_lines.extend(self.network_warning([self._derivative_market_info])) @@ -447,7 +448,10 @@ def did_expire_order(self, expired_event): def did_complete_funding_payment(self, funding_payment_completed_event): # Excute second arbitrage if necessary (even spread hasn't reached min convergence) - if len(self.deriv_position) > 0 and self.ready_for_new_arb_trades(): + if len(self.deriv_position) > 0 and \ + self._all_markets_ready and \ + self.current_proposal and \ + self.ready_for_new_arb_trades(): self.apply_slippage_buffers(self.current_proposal) self.apply_budget_constraint(self.current_proposal) funding_msg = "Executing second arbitrage after funding payment is received" From 1b6aa7ad734322a28b573e62f88f8dabf3d7d4f0 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Tue, 2 Mar 2021 14:55:19 +0800 Subject: [PATCH 62/73] (add) remaining Probit connector unit test cases --- .../probit_api_user_stream_data_source.py | 2 +- .../exchange/probit/connector_test.sqlite | Bin 0 -> 69632 bytes .../exchange/probit/test_probit_exchange.py | 456 ++++++++++++++++++ .../probit/test_probit_order_book_tracker.py | 115 +++++ .../probit/test_probit_user_stream_tracker.py | 41 ++ 5 files changed, 613 insertions(+), 1 deletion(-) create mode 100644 test/connector/exchange/probit/connector_test.sqlite create mode 100644 test/connector/exchange/probit/test_probit_exchange.py create mode 100644 test/connector/exchange/probit/test_probit_order_book_tracker.py create mode 100644 test/connector/exchange/probit/test_probit_user_stream_tracker.py diff --git a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py index b0e9f185de..f8ef3c61c8 100644 --- a/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py +++ b/hummingbot/connector/exchange/probit/probit_api_user_stream_data_source.py @@ -73,7 +73,7 @@ async def _authenticate(self, ws: websockets.WebSocketClientProtocol): Authenticates user to websocket """ try: - auth_payload: Dict[str, Any] = self._probit_auth.get_ws_auth_payload() + auth_payload: Dict[str, Any] = await self._probit_auth.get_ws_auth_payload() await ws.send(ujson.dumps(auth_payload, escape_forward_slashes=False)) auth_resp = await ws.recv() auth_resp: Dict[str, Any] = ujson.loads(auth_resp) diff --git a/test/connector/exchange/probit/connector_test.sqlite b/test/connector/exchange/probit/connector_test.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..86198ba15689734b16c2f1eb34c454628d798acf GIT binary patch literal 69632 zcmeI)(Qn#D90zbaiQ__=kSdF;w9RUPrWG}GkT6C!b#0)eq9sWSL~Ar@mJ=?iTY@Q= zwo%px)J@wTv6uZ7`xB;Vf5H0JPU}Ovm%U9|A9g-tFxZf(TBVkJEh3ydAO79v&WCL{ z3O93_LF661wx=5M5;x89Ja<`^IgYM!+zI+;FH>}7%>IWy^N#g?tJB>1OTSG9{^q7f zf8_$#0zXe_$1hKPKJnv(;MaVAh(C5iXA1pc`O;00drFpgA%vO6d^aJf~Gk zRwf0S?gwc!BCAx{X{ySSuSSe?An#gEqOp0d3{X|Itm8m zv-Zm`$nDn;H}3>QDHP%#pEA^~GI11etWLLfT3k-$GQF?r_h}^>PBC*elb6@li}Kn= zF1H~2efI9;V&)c2b>y^nFQ>9sY0<7sT5noUccB~k?CNrUQ@)VW-Xm5v|^=orv?1s=E*5h+Bm~ER~f6ZqwJ}*yTnko)H+erdY!Je zW3%XsHMK-mv~oGn1^-`)vKDBUr!iPNTaY_@>uQOUlcKaqgUj{=XV4I|%C1sXwOZfs zruu|8sBqr^>-&3KdbumOk>jFti3Yc12e++PcC_8T@l5u{k7_pK*#=^@qM%67o{_2`0u}D$5OlL+ba{T z7+S|sJDv88zDH3ni2J~((>6B9?!G(6ir1ZPwXZw9(=pw7jyk(D+Xs!;0X<+GFb^)p z*sl`zOIoG8^Qtv%o5(A<_2urNL*3IG6~iN=OzxAKx=TFADauEq*dau|R-l8XbD$y* zw{NSJT^b|Xh7NSDv8<}sHAB-YEXN#^{*Q5r_leR)iEn1uW-Ij@DOu%@c}F*>oM(LY0O{p+dYrz;(AYM+-5ke{C!FXV+JIu7 zkvX>aqT_?{Vh!#*==d zQ(3JWO24xkmf5J5=t$n_9$xXbha|mqve+JY4jqa7;jl1njw>ANn)Q2SdMT->4MS)D zE36+VG4?dx{A$!XF&t$bWtE*MndhLyu{ze>X&gCr*YP?rJiPZ<=-RnzfQywSsrae$XA1cq8lDGb>8Z zL_ZoCm(uh78o~}jiP>wT^djS!d7&{$|M-lnt#TMZETu}Z2E@z|57sf$%XTaW|}n-#7vByk4B@hXw*z456O1JpmPqaY&MIO zZR3^6Cg!wr>1;D=z9~(qG|F_YWzbBbQ#?!UZQAoiSm=6{RDzElS>OM^!v*fVktW8A zLI45~fB*y_009U<00Izz00fShKy!vakq&)0H$O8gghZcsPT*z-+}?295@DQ!fuzWJMKnYI+=L(@^m_K`Po0ucj%I8?6aR?NSi;*LQ2;5YuQ}R`u_h@ zF7Wh-H3zB-0SG_<0uX=z1Rwwb2tWV=5Ev$b_eayd0k;{f>;D&A;KeXi8g+pH1Rwwb z2tWV=5P$##AOHaf994l?VQyCNSzjOMd}qMA{(sH|o*z{g)Efd2fB*y_009U<00Izz z00bZ~ECO@F+`N75Z%-Yt<^WjN|1Y_~%VDW2>Hz@=KmY;|fB*y_009U<00Iy=(gN=Z zb2E;c0M6@weEF00Izz00bZa0SG_<0uX?} zPzm7rf2f)hm4N^RAOHafKmY;|fB*y_009Vi3YfqD#|M7l=mQG`AOHafKmY;|fB*y_ z009U<00M8jz{kA6rG;~;NUE5QY!t2(Be6s*o=7BP(ZzT&8ILEdAN)@>_Sb7AQcKr} vYLJro`9B|c!qEp72tWV=5P$##AOHafKmY;|fB*#EIe`zx(t_{M+X4Rqh71df literal 0 HcmV?d00001 diff --git a/test/connector/exchange/probit/test_probit_exchange.py b/test/connector/exchange/probit/test_probit_exchange.py new file mode 100644 index 0000000000..ccf5626557 --- /dev/null +++ b/test/connector/exchange/probit/test_probit_exchange.py @@ -0,0 +1,456 @@ +from os.path import join, realpath +import sys; sys.path.insert(0, realpath(join(__file__, "../../../../../"))) + +import asyncio +import conf +import contextlib +import logging +import math +import os +import unittest +import time + +from decimal import Decimal +from typing import List + +from hummingbot.core.clock import Clock, ClockMode +from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL +from hummingbot.core.utils.async_utils import safe_gather, safe_ensure_future +from hummingbot.core.event.event_logger import EventLogger +from hummingbot.core.event.events import ( + BuyOrderCompletedEvent, + BuyOrderCreatedEvent, + MarketEvent, + OrderFilledEvent, + OrderType, + SellOrderCompletedEvent, + SellOrderCreatedEvent, + OrderCancelledEvent +) +from hummingbot.model.sql_connection_manager import ( + SQLConnectionManager, + SQLConnectionType +) +from hummingbot.model.market_state import MarketState +from hummingbot.model.order import Order +from hummingbot.model.trade_fill import TradeFill +from hummingbot.connector.markets_recorder import MarketsRecorder +from hummingbot.connector.exchange.probit.probit_exchange import ProbitExchange + + +logging.basicConfig(level=METRICS_LOG_LEVEL) +API_KEY = conf.probit_api_key +API_SECRET = conf.probit_secret_key +DOMAIN = "com" + + +class ProbitExchangeUnitTest(unittest.TestCase): + events: List[MarketEvent] = [ + MarketEvent.BuyOrderCompleted, + MarketEvent.SellOrderCompleted, + MarketEvent.OrderFilled, + MarketEvent.TransactionFailure, + MarketEvent.BuyOrderCreated, + MarketEvent.SellOrderCreated, + MarketEvent.OrderCancelled, + MarketEvent.OrderFailure + ] + connector: ProbitExchange + event_logger: EventLogger + trading_pair = "BTC-USDT" + base_token, quote_token = trading_pair.split("-") + stack: contextlib.ExitStack + + @classmethod + def setUpClass(cls): + + cls.ev_loop = asyncio.get_event_loop() + + cls.clock: Clock = Clock(ClockMode.REALTIME) + cls.connector: ProbitExchange = ProbitExchange( + probit_api_key=API_KEY, + probit_secret_key=API_SECRET, + trading_pairs=[cls.trading_pair], + trading_required=True, + domain=DOMAIN + ) + print("Initializing Probit market... this will take about a minute.") + cls.clock.add_iterator(cls.connector) + cls.stack: contextlib.ExitStack = contextlib.ExitStack() + cls._clock = cls.stack.enter_context(cls.clock) + cls.ev_loop.run_until_complete(cls.wait_til_ready()) + print("Ready.") + + @classmethod + def tearDownClass(cls) -> None: + cls.stack.close() + + @classmethod + async def wait_til_ready(cls, connector = None): + if connector is None: + connector = cls.connector + while True: + now = time.time() + next_iteration = now // 1.0 + 1 + if connector.ready: + break + else: + await cls._clock.run_til(next_iteration) + await asyncio.sleep(1.0) + + def setUp(self): + self.db_path: str = realpath(join(__file__, "../connector_test.sqlite")) + try: + os.unlink(self.db_path) + except FileNotFoundError: + pass + + self.event_logger = EventLogger() + for event_tag in self.events: + self.connector.add_listener(event_tag, self.event_logger) + + def tearDown(self): + for event_tag in self.events: + self.connector.remove_listener(event_tag, self.event_logger) + self.event_logger = None + + async def run_parallel_async(self, *tasks): + future: asyncio.Future = safe_ensure_future(safe_gather(*tasks)) + while not future.done(): + now = time.time() + next_iteration = now // 1.0 + 1 + await self._clock.run_til(next_iteration) + await asyncio.sleep(1.0) + return future.result() + + def run_parallel(self, *tasks): + return self.ev_loop.run_until_complete(self.run_parallel_async(*tasks)) + + def test_estimate_fee(self): + maker_fee = self.connector.estimate_fee_pct(True) + self.assertAlmostEqual(maker_fee, Decimal("0.001")) + taker_fee = self.connector.estimate_fee_pct(False) + self.assertAlmostEqual(taker_fee, Decimal("0.001")) + + def _cancel_order(self, cl_order_id): + self.connector.cancel(self.trading_pair, cl_order_id) + + def test_limit_buy_and_sell(self): + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.05") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + quote_bal = self.connector.get_available_balance(self.quote_token) + base_bal = self.connector.get_available_balance(self.base_token) + + order_id = self.connector.buy(trading_pair=self.trading_pair, + amount=amount, + order_type=OrderType.LIMIT, + price=price, + ) + order_completed_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(2)) + trade_events = [t for t in self.event_logger.event_log if isinstance(t, OrderFilledEvent)] + base_amount_traded = sum(t.amount for t in trade_events) + quote_amount_traded = sum(t.amount * t.price for t in trade_events) + + self.assertTrue([evt.order_type == OrderType.LIMIT for evt in trade_events]) + self.assertEqual(order_id, order_completed_event.order_id) + self.assertEqual(amount, order_completed_event.base_asset_amount) + self.assertEqual("BTC", order_completed_event.base_asset) + self.assertEqual("USDT", order_completed_event.quote_asset) + self.assertAlmostEqual(base_amount_traded, order_completed_event.base_asset_amount) + self.assertAlmostEqual(quote_amount_traded, order_completed_event.quote_asset_amount) + self.assertGreater(order_completed_event.fee_amount, Decimal(0)) + self.assertTrue(any([isinstance(event, BuyOrderCreatedEvent) and event.order_id == order_id + for event in self.event_logger.event_log])) + + # check available quote balance gets updated, we need to wait a bit for the balance message to arrive + expected_quote_bal = quote_bal - quote_amount_traded + self._mock_ws_bal_update(self.quote_token, expected_quote_bal) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.assertAlmostEqual(expected_quote_bal, self.connector.get_available_balance(self.quote_token)) + + # Reset the logs + self.event_logger.clear() + + # Try to sell back the same amount to the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.95") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + order_id = self.connector.sell(trading_pair=self.trading_pair, + amount=amount, + order_type=OrderType.LIMIT, + price=price, + ) + order_completed_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCompletedEvent)) + trade_events = [t for t in self.event_logger.event_log if isinstance(t, OrderFilledEvent)] + base_amount_traded = sum(t.amount for t in trade_events) + quote_amount_traded = sum(t.amount * t.price for t in trade_events) + + self.assertTrue([evt.order_type == OrderType.LIMIT for evt in trade_events]) + self.assertEqual(order_id, order_completed_event.order_id) + self.assertEqual(amount, order_completed_event.base_asset_amount) + self.assertEqual("BTC", order_completed_event.base_asset) + self.assertEqual("USDT", order_completed_event.quote_asset) + self.assertAlmostEqual(base_amount_traded, order_completed_event.base_asset_amount) + self.assertAlmostEqual(quote_amount_traded, order_completed_event.quote_asset_amount) + self.assertGreater(order_completed_event.fee_amount, Decimal(0)) + self.assertTrue(any([isinstance(event, SellOrderCreatedEvent) and event.order_id == order_id + for event in self.event_logger.event_log])) + + # check available base balance gets updated, we need to wait a bit for the balance message to arrive + expected_base_bal = base_bal + self._mock_ws_bal_update(self.base_token, expected_base_bal) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + self.assertAlmostEqual(expected_base_bal, self.connector.get_available_balance(self.base_token), 5) + + def test_limit_makers_unfilled(self): + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.8") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + quote_bal = self.connector.get_available_balance(self.quote_token) + + cl_order_id = self.connector.buy(self.trading_pair, amount, OrderType.LIMIT_MAKER, price) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + # check available quote balance gets updated, we need to wait a bit for the balance message to arrive + expected_quote_bal = quote_bal - (price * amount) + self._mock_ws_bal_update(self.quote_token, expected_quote_bal) + self.ev_loop.run_until_complete(asyncio.sleep(2)) + self.assertAlmostEqual(expected_quote_bal, self.connector.get_available_balance(self.quote_token)) + self._cancel_order(cl_order_id) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.2") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + cl_order_id = self.connector.sell(self.trading_pair, amount, OrderType.LIMIT_MAKER, price) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + self._cancel_order(cl_order_id) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + def _mock_ws_bal_update(self, token, available): + # TODO: Determine best way to test balance via ws + pass + + def test_limit_maker_rejections(self): + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.2") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + cl_order_id = self.connector.buy(self.trading_pair, amount, OrderType.LIMIT_MAKER, price) + + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + price = self.connector.get_price(self.trading_pair, False) * Decimal("0.8") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + cl_order_id = self.connector.sell(self.trading_pair, amount, OrderType.LIMIT_MAKER, price) + + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + self.assertEqual(cl_order_id, event.order_id) + + def test_cancel_all(self): + bid_price = self.connector.get_price(self.trading_pair, True) + ask_price = self.connector.get_price(self.trading_pair, False) + bid_price = self.connector.quantize_order_price(self.trading_pair, bid_price * Decimal("0.7")) + ask_price = self.connector.quantize_order_price(self.trading_pair, ask_price * Decimal("1.5")) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + buy_id = self._place_order(True, amount, OrderType.LIMIT, bid_price, 1) + sell_id = self._place_order(False, amount, OrderType.LIMIT, ask_price, 2) + + self.ev_loop.run_until_complete(asyncio.sleep(1)) + asyncio.ensure_future(self.connector.cancel_all(5)) + self.ev_loop.run_until_complete(asyncio.sleep(5)) + cancel_events = [t for t in self.event_logger.event_log if isinstance(t, OrderCancelledEvent)] + self.assertEqual({buy_id, sell_id}, {o.order_id for o in cancel_events}) + + def test_order_price_precision(self): + bid_price: Decimal = self.connector.get_price(self.trading_pair, True) + ask_price: Decimal = self.connector.get_price(self.trading_pair, False) + mid_price: Decimal = (bid_price + ask_price) / 2 + amount: Decimal = Decimal("0.000123456") + + # Make sure there's enough balance to make the limit orders. + self.assertGreater(self.connector.get_balance("BTC"), Decimal("0.001")) + self.assertGreater(self.connector.get_balance("USDT"), Decimal("10")) + + # Intentionally set some prices with too many decimal places s.t. they + # need to be quantized. Also, place them far away from the mid-price s.t. they won't + # get filled during the test. + bid_price = mid_price * Decimal("0.9333192292111341") + ask_price = mid_price * Decimal("1.0492431474884933") + + cl_order_id_1 = self.connector.buy(trading_pair=self.trading_pair, + amount=amount, + order_type=OrderType.LIMIT, + price=bid_price, + ) + + # Wait for the order created event and examine the order made + self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + order = self.connector.in_flight_orders[cl_order_id_1] + quantized_bid_price = self.connector.quantize_order_price(self.trading_pair, bid_price) + quantized_bid_size = self.connector.quantize_order_amount(self.trading_pair, amount) + self.assertEqual(quantized_bid_price, order.price) + self.assertEqual(quantized_bid_size, order.amount) + + # Test ask order + cl_order_id_2 = self.connector.sell(trading_pair=self.trading_pair, + amount=amount, + order_type=OrderType.LIMIT, + price=bid_price, + ) + + # Wait for the order created event and examine and order made + self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCreatedEvent)) + order = self.connector.in_flight_orders[cl_order_id_2] + quantized_ask_price = self.connector.quantize_order_price(self.trading_pair, Decimal(ask_price)) + quantized_ask_size = self.connector.quantize_order_amount(self.trading_pair, Decimal(amount)) + self.assertEqual(quantized_ask_price, order.price) + self.assertEqual(quantized_ask_size, order.amount) + + self._cancel_order(cl_order_id_1) + self._cancel_order(cl_order_id_2) + + def test_orders_saving_and_restoration(self): + config_path = "test_config" + strategy_name = "test_strategy" + sql = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, db_path=self.db_path) + order_id = None + recorder = MarketsRecorder(sql, [self.connector], config_path, strategy_name) + recorder.start() + + try: + self.connector._in_flight_orders.clear() + self.assertEqual(0, len(self.connector.tracking_states)) + + # Try to put limit buy order for 0.02 ETH worth of ZRX, and watch for order creation event. + current_bid_price: Decimal = self.connector.get_price(self.trading_pair, True) + price: Decimal = current_bid_price * Decimal("0.8") + price = self.connector.quantize_order_price(self.trading_pair, price) + + amount: Decimal = Decimal("0.0001") + amount = self.connector.quantize_order_amount(self.trading_pair, amount) + + cl_order_id = self.connector.buy(self.trading_pair, amount, OrderType.LIMIT_MAKER, price) + order_created_event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCreatedEvent)) + self.assertEqual(cl_order_id, order_created_event.order_id) + + # Verify tracking states + self.assertEqual(1, len(self.connector.tracking_states)) + self.assertEqual(cl_order_id, list(self.connector.tracking_states.keys())[0]) + + # Verify orders from recorder + recorded_orders: List[Order] = recorder.get_orders_for_config_and_market(config_path, self.connector) + self.assertEqual(1, len(recorded_orders)) + self.assertEqual(cl_order_id, recorded_orders[0].id) + + # Verify saved market states + saved_market_states: MarketState = recorder.get_market_states(config_path, self.connector) + self.assertIsNotNone(saved_market_states) + self.assertIsInstance(saved_market_states.saved_state, dict) + self.assertGreater(len(saved_market_states.saved_state), 0) + + # Close out the current market and start another market. + self.connector.stop(self._clock) + self.ev_loop.run_until_complete(asyncio.sleep(5)) + self.clock.remove_iterator(self.connector) + for event_tag in self.events: + self.connector.remove_listener(event_tag, self.event_logger) + new_connector = ProbitExchange(API_KEY, API_SECRET, [self.trading_pair], True) + for event_tag in self.events: + new_connector.add_listener(event_tag, self.event_logger) + recorder.stop() + recorder = MarketsRecorder(sql, [new_connector], config_path, strategy_name) + recorder.start() + saved_market_states = recorder.get_market_states(config_path, new_connector) + self.clock.add_iterator(new_connector) + self.assertEqual(0, len(new_connector.limit_orders)) + self.assertEqual(0, len(new_connector.tracking_states)) + new_connector.restore_tracking_states(saved_market_states.saved_state) + self.assertEqual(1, len(new_connector.limit_orders)) + self.assertEqual(1, len(new_connector.tracking_states)) + + # Cancel the order and verify that the change is saved. + self._cancel_order(cl_order_id) + self.ev_loop.run_until_complete(self.event_logger.wait_for(OrderCancelledEvent)) + order_id = None + self.assertEqual(0, len(new_connector.limit_orders)) + self.assertEqual(0, len(new_connector.tracking_states)) + saved_market_states = recorder.get_market_states(config_path, new_connector) + self.assertEqual(0, len(saved_market_states.saved_state)) + finally: + if order_id is not None: + self.connector.cancel(self.trading_pair, cl_order_id) + self.run_parallel(self.event_logger.wait_for(OrderCancelledEvent)) + + recorder.stop() + os.unlink(self.db_path) + + def test_update_last_prices(self): + # This is basic test to see if order_book last_trade_price is initiated and updated. + for order_book in self.connector.order_books.values(): + for _ in range(5): + self.ev_loop.run_until_complete(asyncio.sleep(1)) + print(order_book.last_trade_price) + self.assertFalse(math.isnan(order_book.last_trade_price)) + + def test_filled_orders_recorded(self): + config_path: str = "test_config" + strategy_name: str = "test_strategy" + sql = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, db_path=self.db_path) + order_id = None + recorder = MarketsRecorder(sql, [self.connector], config_path, strategy_name) + recorder.start() + + try: + # Try to buy some token from the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("1.05") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + + order_id = self.connector.buy(trading_pair=self.trading_pair, + amount=amount, + order_type=OrderType.LIMIT, + price=price, + ) + self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + + # Reset the logs + self.event_logger.clear() + + # Try to sell back the same amount to the exchange, and watch for completion event. + price = self.connector.get_price(self.trading_pair, True) * Decimal("0.95") + price = self.connector.quantize_order_price(self.trading_pair, price) + amount = self.connector.quantize_order_amount(self.trading_pair, Decimal("0.0001")) + order_id = self.connector.sell(trading_pair=self.trading_pair, + amount=amount, + order_type=OrderType.LIMIT, + price=price, + ) + self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCompletedEvent)) + + # Query the persisted trade logs + trade_fills: List[TradeFill] = recorder.get_trades_for_config(config_path) + self.assertGreaterEqual(len(trade_fills), 2) + buy_fills: List[TradeFill] = [t for t in trade_fills if t.trade_type == "BUY"] + sell_fills: List[TradeFill] = [t for t in trade_fills if t.trade_type == "SELL"] + self.assertGreaterEqual(len(buy_fills), 1) + self.assertGreaterEqual(len(sell_fills), 1) + + order_id = None + + finally: + if order_id is not None: + self.connector.cancel(self.trading_pair, order_id) + self.run_parallel(self.event_logger.wait_for(OrderCancelledEvent)) + + recorder.stop() + os.unlink(self.db_path) diff --git a/test/connector/exchange/probit/test_probit_order_book_tracker.py b/test/connector/exchange/probit/test_probit_order_book_tracker.py new file mode 100644 index 0000000000..22899e0e2f --- /dev/null +++ b/test/connector/exchange/probit/test_probit_order_book_tracker.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python + +from os.path import join, realpath +import sys; sys.path.insert(0, realpath(join(__file__, "../../../../../"))) + +import asyncio +import logging +import math +import time +import unittest + +from typing import ( + Dict, + Optional, + List, +) + +from hummingbot.core.event.event_logger import EventLogger +from hummingbot.core.event.events import OrderBookEvent, OrderBookTradeEvent, TradeType +from hummingbot.connector.exchange.probit.probit_order_book_tracker import ProbitOrderBookTracker +from hummingbot.connector.exchange.probit.probit_api_order_book_data_source import ProbitAPIOrderBookDataSource +from hummingbot.core.data_type.order_book import OrderBook + + +class ProbitOrderBookTrackerUnitTest(unittest.TestCase): + order_book_tracker: Optional[ProbitOrderBookTracker] = None + events: List[OrderBookEvent] = [ + OrderBookEvent.TradeEvent + ] + trading_pairs: List[str] = [ + "BTC-USDT", + "ETH-USDT", + ] + + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + cls.order_book_tracker: ProbitOrderBookTracker = ProbitOrderBookTracker(cls.trading_pairs) + cls.order_book_tracker.start() + cls.ev_loop.run_until_complete(cls.wait_til_tracker_ready()) + + @classmethod + async def wait_til_tracker_ready(cls): + while True: + if len(cls.order_book_tracker.order_books) > 0: + print("Initialized real-time order books.") + return + await asyncio.sleep(1) + + async def run_parallel_async(self, *tasks, timeout=None): + future: asyncio.Future = asyncio.ensure_future(asyncio.gather(*tasks)) + timer = 0 + while not future.done(): + if timeout and timer > timeout: + raise Exception("Timeout running parallel async tasks in tests") + timer += 1 + now = time.time() + _next_iteration = now // 1.0 + 1 # noqa: F841 + await asyncio.sleep(1.0) + return future.result() + + def run_parallel(self, *tasks): + return self.ev_loop.run_until_complete(self.run_parallel_async(*tasks)) + + def setUp(self): + self.event_logger = EventLogger() + for event_tag in self.events: + for trading_pair, order_book in self.order_book_tracker.order_books.items(): + order_book.add_listener(event_tag, self.event_logger) + + def test_order_book_trade_event_emission(self): + """ + Tests if the order book tracker is able to retrieve order book trade message from exchange and emit order book + trade events after correctly parsing the trade messages + """ + self.run_parallel(self.event_logger.wait_for(OrderBookTradeEvent)) + for ob_trade_event in self.event_logger.event_log: + self.assertTrue(type(ob_trade_event) == OrderBookTradeEvent) + self.assertTrue(ob_trade_event.trading_pair in self.trading_pairs) + self.assertTrue(type(ob_trade_event.timestamp) in [float, int]) + self.assertTrue(type(ob_trade_event.amount) == float) + self.assertTrue(type(ob_trade_event.price) == float) + self.assertTrue(type(ob_trade_event.type) == TradeType) + # datetime is in seconds + self.assertTrue(math.ceil(math.log10(ob_trade_event.timestamp)) == 10) + self.assertTrue(ob_trade_event.amount > 0) + self.assertTrue(ob_trade_event.price > 0) + + def test_tracker_integrity(self): + # Wait 10 seconds to process some diffs. + self.ev_loop.run_until_complete(asyncio.sleep(10.0)) + order_books: Dict[str, OrderBook] = self.order_book_tracker.order_books + eth_usdt: OrderBook = order_books["ETH-USDT"] + self.assertIsNot(eth_usdt.last_diff_uid, 0) + self.assertGreaterEqual(eth_usdt.get_price_for_volume(True, 10).result_price, + eth_usdt.get_price(True)) + self.assertLessEqual(eth_usdt.get_price_for_volume(False, 10).result_price, + eth_usdt.get_price(False)) + + def test_api_get_last_traded_prices(self): + prices = self.ev_loop.run_until_complete( + ProbitAPIOrderBookDataSource.get_last_traded_prices(["BTC-USDT", "ETH-USDT"])) + for key, value in prices.items(): + print(f"{key} last_trade_price: {value}") + self.assertGreater(prices["BTC-USDT"], 30000) + self.assertGreater(prices["ETH-USDT"], 1000) + + +def main(): + logging.basicConfig(level=logging.INFO) + unittest.main() + + +if __name__ == "__main__": + main() diff --git a/test/connector/exchange/probit/test_probit_user_stream_tracker.py b/test/connector/exchange/probit/test_probit_user_stream_tracker.py new file mode 100644 index 0000000000..b17dc67934 --- /dev/null +++ b/test/connector/exchange/probit/test_probit_user_stream_tracker.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +from os.path import join, realpath +import sys; sys.path.insert(0, realpath(join(__file__, "../../../"))) + +import asyncio +import conf +import logging +import unittest + + +from hummingbot.connector.exchange.probit.probit_user_stream_tracker import ProbitUserStreamTracker +from hummingbot.connector.exchange.probit.probit_auth import ProbitAuth +from hummingbot.core.utils.async_utils import safe_ensure_future + + +class ProbitUserStreamTrackerUnitTest(unittest.TestCase): + api_key = conf.probit_api_key + api_secret = conf.probit_secret_key + + @classmethod + def setUpClass(cls): + cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() + cls.probit_auth = ProbitAuth(cls.api_key, cls.api_secret) + cls.trading_pairs = ["PROB-USDT"] + cls.user_stream_tracker: ProbitUserStreamTracker = ProbitUserStreamTracker( + probit_auth=cls.probit_auth, trading_pairs=cls.trading_pairs) + cls.user_stream_tracker_task: asyncio.Task = safe_ensure_future(cls.user_stream_tracker.start()) + + def test_user_stream(self): + # Wait process some msgs. + self.ev_loop.run_until_complete(asyncio.sleep(120.0)) + print(self.user_stream_tracker.user_stream) + + +def main(): + logging.basicConfig(level=logging.INFO) + unittest.main() + + +if __name__ == "__main__": + main() From 02bf78836768e989f1e29ef599320dd69d5b1316 Mon Sep 17 00:00:00 2001 From: Nullably <37262506+Nullably@users.noreply.github.com> Date: Tue, 2 Mar 2021 15:07:14 +0800 Subject: [PATCH 63/73] (feat) update source of miner data --- hummingbot/connector/parrot.py | 65 +++++++++++-------- .../liquidity_mining/liquidity_mining.py | 5 +- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/hummingbot/connector/parrot.py b/hummingbot/connector/parrot.py index e7ada091ef..cf9979dc2a 100644 --- a/hummingbot/connector/parrot.py +++ b/hummingbot/connector/parrot.py @@ -1,11 +1,13 @@ import aiohttp +import asyncio from typing import List, Dict from dataclasses import dataclass from decimal import Decimal +import logging from hummingbot.connector.exchange.binance.binance_utils import convert_from_exchange_trading_pair from hummingbot.core.utils.async_utils import safe_gather -PARROT_MINER_BASE_URL = "https://papi-development.hummingbot.io/v1/mining_data/" +PARROT_MINER_BASE_URL = "https://papi.hummingbot.io/v1/mining_data/" s_decimal_0 = Decimal("0") @@ -18,37 +20,41 @@ class CampaignSummary: spread_max: Decimal = s_decimal_0 payout_asset: str = "" liquidity: Decimal = s_decimal_0 + liquidity_usd: Decimal = s_decimal_0 active_bots: int = 0 - reward_per_day: Decimal = s_decimal_0 + reward_per_wk: Decimal = s_decimal_0 apy: Decimal = s_decimal_0 async def get_campaign_summary(exchange: str, trading_pairs: List[str] = []) -> Dict[str, CampaignSummary]: - campaigns = await get_active_campaigns(exchange, trading_pairs) - tasks = [get_market_snapshots(m_id) for m_id in campaigns] - results = await safe_gather(*tasks, return_exceptions=True) - for result in results: - if isinstance(result, Exception): - raise result - if result["items"]: - snapshot = result["items"][0] - market_id = int(snapshot["market_id"]) - campaign = campaigns[market_id] - campaign.apy = Decimal(snapshot["annualized_return"]) / Decimal("100") - reward = snapshot["payout_summary"]["open_volume"]["reward"] - if campaign.payout_asset in reward["ask"]: - campaign.reward_per_day = Decimal(str(reward["ask"][campaign.payout_asset])) - if campaign.payout_asset in reward["bid"]: - campaign.reward_per_day += Decimal(str(reward["bid"][campaign.payout_asset])) - oov = snapshot["summary_stats"]["open_volume"] - campaign.liquidity = Decimal(oov["oov_ask"]) + Decimal(oov["oov_bid"]) - campaign.active_bots = int(oov["bots"]) - return {c.trading_pair: c for c in campaigns.values()} + results = {} + try: + campaigns = await get_active_campaigns(exchange, trading_pairs) + tasks = [get_market_snapshots(m_id) for m_id in campaigns] + snapshots = await safe_gather(*tasks, return_exceptions=True) + for snapshot in snapshots: + if isinstance(snapshot, Exception): + raise snapshot + if snapshot["items"]: + snapshot = snapshot["items"][0] + market_id = int(snapshot["market_id"]) + campaign = campaigns[market_id] + campaign.apy = Decimal(snapshot["annualized_return"]) + oov = snapshot["summary_stats"]["open_volume"] + campaign.liquidity = Decimal(oov["oov_eligible_ask"]) + Decimal(oov["oov_eligible_bid"]) + campaign.liquidity_usd = campaign.liquidity * Decimal(oov["base_asset_usd_rate"]) + campaign.active_bots = int(oov["bots"]) + results = {c.trading_pair: c for c in campaigns.values()} + except asyncio.CancelledError: + raise + except Exception: + logging.getLogger(__name__).error("Unexpected error while requesting data from Hummingbot API.", exc_info=True) + return results async def get_market_snapshots(market_id: int): async with aiohttp.ClientSession() as client: - url = f"{PARROT_MINER_BASE_URL}market_snapshots/{market_id}?aggregate=1d" + url = f"{PARROT_MINER_BASE_URL}market_snapshots/{market_id}?aggregate=1m" resp = await client.get(url) resp_json = await resp.json() return resp_json @@ -65,6 +71,9 @@ async def get_active_campaigns(exchange: str, trading_pairs: List[str] = []) -> if market["exchange_name"] != exchange: continue t_pair = market["trading_pair"] + # So far we have only 2 exchanges for mining Binance and Kucoin, Kucoin doesn't require conversion. + # In the future we should create a general approach for this, e.g. move all convert trading pair fn to + # utils.py and import the function dynamically in hummingbot/client/settings.py if exchange == "binance": t_pair = convert_from_exchange_trading_pair(t_pair) if trading_pairs and t_pair not in trading_pairs: @@ -75,9 +84,11 @@ async def get_active_campaigns(exchange: str, trading_pairs: List[str] = []) -> campaign.exchange_name = market["exchange_name"] campaigns[campaign.market_id] = campaign for bounty_period in campaign_retval["bounty_periods"]: - for payout_parameter in bounty_period["payout_parameters"]: - market_id = int(payout_parameter["market_id"]) + for payout in bounty_period["payout_parameters"]: + market_id = int(payout["market_id"]) if market_id in campaigns: - campaigns[market_id].spread_max = Decimal(str(payout_parameter["spread_max"])) / Decimal("100") - campaigns[market_id].payout_asset = payout_parameter["payout_asset"] + campaigns[market_id].reward_per_wk = Decimal(str(payout["bid_budget"])) + \ + Decimal(str(payout["ask_budget"])) + campaigns[market_id].spread_max = Decimal(str(payout["spread_max"])) / Decimal("100") + campaigns[market_id].payout_asset = payout["payout_asset"] return campaigns diff --git a/hummingbot/strategy/liquidity_mining/liquidity_mining.py b/hummingbot/strategy/liquidity_mining/liquidity_mining.py index e2f25558cb..1b262376c2 100644 --- a/hummingbot/strategy/liquidity_mining/liquidity_mining.py +++ b/hummingbot/strategy/liquidity_mining/liquidity_mining.py @@ -195,13 +195,12 @@ async def miner_status_df(self) -> pd.DataFrame: columns = ["Market", "Payout", "Reward/wk", "Liquidity", "Yield/yr", "Max spread"] campaigns = await get_campaign_summary(self._exchange.display_name, list(self._market_infos.keys())) for market, campaign in campaigns.items(): - reward_usd = await usd_value(campaign.payout_asset, campaign.reward_per_day * Decimal("7")) - liquidity_usd = await usd_value(market.split('-')[0], campaign.liquidity) + reward_usd = await usd_value(campaign.payout_asset, campaign.reward_per_wk) data.append([ market, campaign.payout_asset, f"${reward_usd:.0f}", - f"${liquidity_usd:.0f}", + f"${campaign.liquidity_usd:.0f}", f"{campaign.apy:.2%}", f"{campaign.spread_max:.2%}%" ]) From e6fbfc0f542a711a9cd42f33093c89778e335fd2 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Tue, 2 Mar 2021 15:24:40 +0800 Subject: [PATCH 64/73] (remove) remove connector_test.sqlite file --- .../exchange/probit/connector_test.sqlite | Bin 69632 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 test/connector/exchange/probit/connector_test.sqlite diff --git a/test/connector/exchange/probit/connector_test.sqlite b/test/connector/exchange/probit/connector_test.sqlite deleted file mode 100644 index 86198ba15689734b16c2f1eb34c454628d798acf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 69632 zcmeI)(Qn#D90zbaiQ__=kSdF;w9RUPrWG}GkT6C!b#0)eq9sWSL~Ar@mJ=?iTY@Q= zwo%px)J@wTv6uZ7`xB;Vf5H0JPU}Ovm%U9|A9g-tFxZf(TBVkJEh3ydAO79v&WCL{ z3O93_LF661wx=5M5;x89Ja<`^IgYM!+zI+;FH>}7%>IWy^N#g?tJB>1OTSG9{^q7f zf8_$#0zXe_$1hKPKJnv(;MaVAh(C5iXA1pc`O;00drFpgA%vO6d^aJf~Gk zRwf0S?gwc!BCAx{X{ySSuSSe?An#gEqOp0d3{X|Itm8m zv-Zm`$nDn;H}3>QDHP%#pEA^~GI11etWLLfT3k-$GQF?r_h}^>PBC*elb6@li}Kn= zF1H~2efI9;V&)c2b>y^nFQ>9sY0<7sT5noUccB~k?CNrUQ@)VW-Xm5v|^=orv?1s=E*5h+Bm~ER~f6ZqwJ}*yTnko)H+erdY!Je zW3%XsHMK-mv~oGn1^-`)vKDBUr!iPNTaY_@>uQOUlcKaqgUj{=XV4I|%C1sXwOZfs zruu|8sBqr^>-&3KdbumOk>jFti3Yc12e++PcC_8T@l5u{k7_pK*#=^@qM%67o{_2`0u}D$5OlL+ba{T z7+S|sJDv88zDH3ni2J~((>6B9?!G(6ir1ZPwXZw9(=pw7jyk(D+Xs!;0X<+GFb^)p z*sl`zOIoG8^Qtv%o5(A<_2urNL*3IG6~iN=OzxAKx=TFADauEq*dau|R-l8XbD$y* zw{NSJT^b|Xh7NSDv8<}sHAB-YEXN#^{*Q5r_leR)iEn1uW-Ij@DOu%@c}F*>oM(LY0O{p+dYrz;(AYM+-5ke{C!FXV+JIu7 zkvX>aqT_?{Vh!#*==d zQ(3JWO24xkmf5J5=t$n_9$xXbha|mqve+JY4jqa7;jl1njw>ANn)Q2SdMT->4MS)D zE36+VG4?dx{A$!XF&t$bWtE*MndhLyu{ze>X&gCr*YP?rJiPZ<=-RnzfQywSsrae$XA1cq8lDGb>8Z zL_ZoCm(uh78o~}jiP>wT^djS!d7&{$|M-lnt#TMZETu}Z2E@z|57sf$%XTaW|}n-#7vByk4B@hXw*z456O1JpmPqaY&MIO zZR3^6Cg!wr>1;D=z9~(qG|F_YWzbBbQ#?!UZQAoiSm=6{RDzElS>OM^!v*fVktW8A zLI45~fB*y_009U<00Izz00fShKy!vakq&)0H$O8gghZcsPT*z-+}?295@DQ!fuzWJMKnYI+=L(@^m_K`Po0ucj%I8?6aR?NSi;*LQ2;5YuQ}R`u_h@ zF7Wh-H3zB-0SG_<0uX=z1Rwwb2tWV=5Ev$b_eayd0k;{f>;D&A;KeXi8g+pH1Rwwb z2tWV=5P$##AOHaf994l?VQyCNSzjOMd}qMA{(sH|o*z{g)Efd2fB*y_009U<00Izz z00bZ~ECO@F+`N75Z%-Yt<^WjN|1Y_~%VDW2>Hz@=KmY;|fB*y_009U<00Iy=(gN=Z zb2E;c0M6@weEF00Izz00bZa0SG_<0uX?} zPzm7rf2f)hm4N^RAOHafKmY;|fB*y_009Vi3YfqD#|M7l=mQG`AOHafKmY;|fB*y_ z009U<00M8jz{kA6rG;~;NUE5QY!t2(Be6s*o=7BP(ZzT&8ILEdAN)@>_Sb7AQcKr} vYLJro`9B|c!qEp72tWV=5P$##AOHafKmY;|fB*#EIe`zx(t_{M+X4Rqh71df From 7f6be282f15b6b182fde272a1fa823f5ccb72840 Mon Sep 17 00:00:00 2001 From: Nullably <37262506+Nullably@users.noreply.github.com> Date: Tue, 2 Mar 2021 17:25:33 +0800 Subject: [PATCH 65/73] (feat) minor fix --- hummingbot/connector/parrot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/connector/parrot.py b/hummingbot/connector/parrot.py index cf9979dc2a..55b3ccad73 100644 --- a/hummingbot/connector/parrot.py +++ b/hummingbot/connector/parrot.py @@ -16,7 +16,7 @@ class CampaignSummary: market_id: int = 0 trading_pair: str = "" - exchange_name: str = 0 + exchange_name: str = "" spread_max: Decimal = s_decimal_0 payout_asset: str = "" liquidity: Decimal = s_decimal_0 From 98919ab47bce625eafb736e509d4d8c58ce16323 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Tue, 2 Mar 2021 17:23:51 +0800 Subject: [PATCH 66/73] (fix) fix conf secret key for ProbitAuthUnitTest --- test/connector/exchange/probit/test_probit_auth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/connector/exchange/probit/test_probit_auth.py b/test/connector/exchange/probit/test_probit_auth.py index 4d6ccdab51..8c8fbef08a 100644 --- a/test/connector/exchange/probit/test_probit_auth.py +++ b/test/connector/exchange/probit/test_probit_auth.py @@ -22,13 +22,13 @@ logging.basicConfig(level=METRICS_LOG_LEVEL) -class ProBitAuthUnitTest(unittest.TestCase): +class ProbitAuthUnitTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() api_key = conf.probit_api_key - secret_key = conf.crypto_com_secret_key + secret_key = conf.probit_secret_key cls.auth: ProbitAuth = ProbitAuth(api_key, secret_key) async def rest_auth(self) -> Dict[str, Any]: From 69ce1b8c0b9b6ca508341f8919347ec650d89559 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Tue, 2 Mar 2021 17:27:48 +0800 Subject: [PATCH 67/73] (remove) remove redundant websocket client initialization --- .../exchange/probit/probit_api_order_book_data_source.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py b/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py index 4e6a8550a4..93e464cd55 100644 --- a/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py +++ b/hummingbot/connector/exchange/probit/probit_api_order_book_data_source.py @@ -130,7 +130,6 @@ async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asynci while True: try: async with websockets.connect(uri=CONSTANTS.WSS_URL.format(self._domain)) as ws: - ws: websockets.WebSocketClientProtocol = ws for trading_pair in self._trading_pairs: params: Dict[str, Any] = { "channel": "marketdata", From 12bd67a893b9a69f09770aa6ff9bede6e2f5c259 Mon Sep 17 00:00:00 2001 From: vic-en Date: Tue, 2 Mar 2021 10:38:08 +0100 Subject: [PATCH 68/73] (fix) fix market 1 validator --- hummingbot/strategy/amm_arb/amm_arb_config_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/strategy/amm_arb/amm_arb_config_map.py b/hummingbot/strategy/amm_arb/amm_arb_config_map.py index f662855f10..d695ec7c55 100644 --- a/hummingbot/strategy/amm_arb/amm_arb_config_map.py +++ b/hummingbot/strategy/amm_arb/amm_arb_config_map.py @@ -70,7 +70,7 @@ def order_amount_prompt() -> str: key="market_1", prompt=market_1_prompt, prompt_on_new=True, - validator=market_1_on_validated, + validator=market_1_validator, on_validated=market_1_on_validated), "connector_2": ConfigVar( key="connector_2", From 5396464921a51aec1b23d30864c5b8a9495a3176 Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Tue, 2 Mar 2021 17:54:47 +0800 Subject: [PATCH 69/73] (add) include better error handling in ProbitExchange --- .../exchange/probit/probit_exchange.py | 54 +++++-------------- 1 file changed, 14 insertions(+), 40 deletions(-) diff --git a/hummingbot/connector/exchange/probit/probit_exchange.py b/hummingbot/connector/exchange/probit/probit_exchange.py index ae3dd1274e..c93b5e76aa 100644 --- a/hummingbot/connector/exchange/probit/probit_exchange.py +++ b/hummingbot/connector/exchange/probit/probit_exchange.py @@ -319,35 +319,6 @@ def _format_trading_rules(self, market_info: Dict[str, Any]) -> Dict[str, Tradin self.logger().error(f"Error parsing the trading pair rule {market}. Skipping.", exc_info=True) return result - async def _get_auth_headers(self, http_client: aiohttp.ClientSession) -> Dict[str, Any]: - if self._probit_auth.token_has_expired: - try: - now: int = int(time.time()) - headers = self._probit_auth.get_headers() - headers.update({ - "Authorization": f"Basic {self._probit_auth.token_payload}" - }) - body = ujson.dumps({ - "grant_type": "client_credentials" - }) - resp = await http_client.post(url=CONSTANTS.TOKEN_URL, - headers=headers, - data=body) - token_resp = await resp.json() - - if resp.status != 200: - raise ValueError(f"Error occurred retrieving new OAuth Token. Response: {token_resp}") - - # POST /token endpoint returns both access_token and expires_in - # Updates _oauth_token_expiration_time - - self._probit_auth.update_expiration_time(now + token_resp["expires_in"]) - self._probit_auth.update_oauth_token(token_resp["access_token"]) - except Exception as e: - raise e - - return self._probit_auth.generate_auth_dict() - async def _api_request(self, method: str, path_url: str, @@ -365,20 +336,23 @@ async def _api_request(self, path_url = path_url.format(self._domain) client = await self._http_client() - if is_auth_required: - headers = await self._probit_auth.get_auth_headers(client) - else: - headers = self._probit_auth.get_headers() + try: + if is_auth_required: + headers = await self._probit_auth.get_auth_headers(client) + else: + headers = self._probit_auth.get_headers() - if method == "GET": - response = await client.get(path_url, headers=headers, params=params) - elif method == "POST": - response = await client.post(path_url, headers=headers, data=ujson.dumps(data)) - else: - raise NotImplementedError(f"{method} HTTP Method not implemented. ") + if method == "GET": + response = await client.get(path_url, headers=headers, params=params) + elif method == "POST": + response = await client.post(path_url, headers=headers, data=ujson.dumps(data)) + else: + raise NotImplementedError(f"{method} HTTP Method not implemented. ") - try: parsed_response = await response.json() + except ValueError as e: + self.logger().error(f"{str(e)}") + raise ValueError(f"Error authenticating request {method} {path_url}. Error: {str(e)}") except Exception as e: raise IOError(f"Error parsing data from {path_url}. Error: {str(e)}") if response.status != 200: From fae9fc057789f7c318c0b2f340f6f8a515282635 Mon Sep 17 00:00:00 2001 From: vic-en Date: Tue, 2 Mar 2021 11:03:23 +0100 Subject: [PATCH 70/73] (fix) add vaidator for amount parameter --- hummingbot/strategy/amm_arb/amm_arb_config_map.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hummingbot/strategy/amm_arb/amm_arb_config_map.py b/hummingbot/strategy/amm_arb/amm_arb_config_map.py index d695ec7c55..2e648e0cb9 100644 --- a/hummingbot/strategy/amm_arb/amm_arb_config_map.py +++ b/hummingbot/strategy/amm_arb/amm_arb_config_map.py @@ -88,6 +88,7 @@ def order_amount_prompt() -> str: key="order_amount", prompt=order_amount_prompt, type_str="decimal", + validator=lambda v: validate_decimal(v, Decimal("0")), prompt_on_new=True), "min_profitability": ConfigVar( key="min_profitability", From efd6e7d669cba9bc3278483e438e3b6625828f6f Mon Sep 17 00:00:00 2001 From: Nicolas Baum Date: Thu, 4 Mar 2021 00:37:12 -0300 Subject: [PATCH 71/73] (fix) Mapped Kraken tickers to Hummingbot names in the balance update process --- hummingbot/connector/exchange/kraken/kraken_exchange.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hummingbot/connector/exchange/kraken/kraken_exchange.pyx b/hummingbot/connector/exchange/kraken/kraken_exchange.pyx index 4176e5f5e1..66fedfe487 100755 --- a/hummingbot/connector/exchange/kraken/kraken_exchange.pyx +++ b/hummingbot/connector/exchange/kraken/kraken_exchange.pyx @@ -230,9 +230,9 @@ cdef class KrakenExchange(ExchangeBase): (base, quote) = self.split_trading_pair(pair) vol_locked = Decimal(order.get("vol", 0)) - Decimal(order.get("vol_exec", 0)) if details.get("type") == "sell": - locked[base] += vol_locked + locked[convert_from_exchange_symbol(base)] += vol_locked elif details.get("type") == "buy": - locked[quote] += vol_locked * Decimal(details.get("price")) + locked[convert_from_exchange_symbol(quote)] += vol_locked * Decimal(details.get("price")) for asset_name, balance in balances.items(): cleaned_name = convert_from_exchange_symbol(asset_name).upper() From 8770d2018287cc8a1a714b4409425cd4ff5c6190 Mon Sep 17 00:00:00 2001 From: vic-en Date: Thu, 4 Mar 2021 10:33:19 +0100 Subject: [PATCH 72/73] (feat) ajusted precision of parameters of an active position --- .../perpetual_finance/perpetual_finance_derivative.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py index e8ea75a72a..73eea1569f 100644 --- a/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py +++ b/hummingbot/connector/derivative/perpetual_finance/perpetual_finance_derivative.py @@ -507,11 +507,11 @@ async def _update_positions(self): positions = await safe_gather(*position_tasks, return_exceptions=True) for trading_pair, position in zip(self._trading_pairs, positions): position = position.get("position", {}) - amount = Decimal(position.get("size")) + amount = self.quantize_order_amount(trading_pair, Decimal(position.get("size"))) if amount != Decimal("0"): position_side = PositionSide.LONG if amount > 0 else PositionSide.SHORT - unrealized_pnl = Decimal(position.get("pnl")) - entry_price = Decimal(position.get("entryPrice")) + unrealized_pnl = self.quantize_order_amount(trading_pair, Decimal(position.get("pnl"))) + entry_price = self.quantize_order_price(trading_pair, Decimal(position.get("entryPrice"))) leverage = self._leverage[trading_pair] self._account_positions[trading_pair] = Position( trading_pair=trading_pair, From f0c402497ada0c1feccafc725b9388165fc2e960 Mon Sep 17 00:00:00 2001 From: vic-en Date: Thu, 4 Mar 2021 11:40:15 +0100 Subject: [PATCH 73/73] (feat) add test cases for perpfi connector --- .../connector/perpfi/test_perfi_connector.py | 209 ++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 test/connector/connector/perpfi/test_perfi_connector.py diff --git a/test/connector/connector/perpfi/test_perfi_connector.py b/test/connector/connector/perpfi/test_perfi_connector.py new file mode 100644 index 0000000000..dc0ec26c16 --- /dev/null +++ b/test/connector/connector/perpfi/test_perfi_connector.py @@ -0,0 +1,209 @@ +from os.path import join, realpath +import sys; sys.path.insert(0, realpath(join(__file__, "../../../../../"))) +import unittest +import unittest.mock +import asyncio +import os +from decimal import Decimal +from typing import List +import contextlib +import time +from hummingbot.core.clock import Clock, ClockMode +from hummingbot.core.event.event_logger import EventLogger +from hummingbot.connector.derivative.perpetual_finance.perpetual_finance_derivative import PerpetualFinanceDerivative +from hummingbot.core.event.events import ( + BuyOrderCompletedEvent, + MarketEvent, + OrderType, + SellOrderCompletedEvent, + MarketOrderFailureEvent, + PositionAction, PositionMode +) +from hummingbot.model.sql_connection_manager import ( + SQLConnectionManager, + SQLConnectionType +) +from hummingbot.model.trade_fill import TradeFill +from hummingbot.connector.markets_recorder import MarketsRecorder +from hummingbot.client.config.global_config_map import global_config_map + +global_config_map['gateway_api_host'].value = "localhost" +global_config_map['gateway_api_port'].value = 5000 +global_config_map.get("ethereum_chain_name").value = "mainnet" + +trading_pair = "SNX-USDC" +leverage = 3 +base, quote = trading_pair.split("-") + + +class PerpetualFinanceDerivativeUnitTest(unittest.TestCase): + event_logger: EventLogger + events: List[MarketEvent] = [ + MarketEvent.BuyOrderCompleted, + MarketEvent.SellOrderCompleted, + MarketEvent.OrderFilled, + MarketEvent.TransactionFailure, + MarketEvent.BuyOrderCreated, + MarketEvent.SellOrderCreated, + MarketEvent.OrderCancelled, + MarketEvent.OrderFailure, + MarketEvent.FundingPaymentCompleted + ] + connector: PerpetualFinanceDerivative + stack: contextlib.ExitStack + + @classmethod + def setUpClass(cls): + cls.ev_loop = asyncio.get_event_loop() + cls.clock: Clock = Clock(ClockMode.REALTIME) + cls.connector: PerpetualFinanceDerivative = PerpetualFinanceDerivative( + [trading_pair], + "PRIVATE_KEY_HERE", + "") + print("Initializing PerpetualFinanceDerivative market... this will take about a minute.") + cls.connector.set_leverage(trading_pair, leverage) + cls.connector.set_position_mode(PositionMode.ONEWAY) + cls.clock.add_iterator(cls.connector) + cls.stack: contextlib.ExitStack = contextlib.ExitStack() + cls._clock = cls.stack.enter_context(cls.clock) + cls.ev_loop.run_until_complete(cls.wait_til_ready()) + print("Ready.") + + @classmethod + def tearDownClass(cls) -> None: + cls.stack.close() + + @classmethod + async def wait_til_ready(cls): + while True: + now = time.time() + next_iteration = now // 1.0 + 1 + if cls.connector.ready: + break + else: + await cls._clock.run_til(next_iteration) + await asyncio.sleep(1.0) + + def setUp(self): + self.db_path: str = realpath(join(__file__, "../connector_test.sqlite")) + try: + os.unlink(self.db_path) + except FileNotFoundError: + pass + self.event_logger = EventLogger() + for event_tag in self.events: + self.connector.add_listener(event_tag, self.event_logger) + + def test_update_balances(self): + all_bals = self.connector.get_all_balances() + for token, bal in all_bals.items(): + print(f"{token}: {bal}") + self.assertIn(quote, all_bals) + self.assertTrue(all_bals["XDAI"] > 0) + + def test_allowances(self): + asyncio.get_event_loop().run_until_complete(self._test_allowances()) + + async def _test_allowances(self): + perfi = self.connector + allowances = await perfi.get_allowances() + print(allowances) + + def test_approve(self): + asyncio.get_event_loop().run_until_complete(self._test_approve()) + + async def _test_approve(self): + perfi = self.connector + ret_val = await perfi.approve_perpetual_finance_spender() + print(ret_val) + + def test_get_quote_price(self): + asyncio.get_event_loop().run_until_complete(self._test_get_quote_price()) + + async def _test_get_quote_price(self): + perfi = self.connector + buy_price = await perfi.get_quote_price(trading_pair, True, Decimal("1")) + self.assertTrue(buy_price > 0) + print(f"buy_price: {buy_price}") + sell_price = await perfi.get_quote_price(trading_pair, False, Decimal("1")) + self.assertTrue(sell_price > 0) + print(f"sell_price: {sell_price}") + self.assertTrue(buy_price != sell_price) + + def test_open_and_close_long_position(self): + perfi = self.connector + amount = Decimal("0.1") + price = Decimal("10") + order_id = perfi.buy(trading_pair, amount, OrderType.LIMIT, price, position_action=PositionAction.OPEN) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCompletedEvent)) + self.assertTrue(event.order_id is not None) + self.assertEqual(order_id, event.order_id) + self.assertEqual(event.base_asset_amount, amount) + print(event.order_id) + + order_id = perfi.sell(trading_pair, amount, OrderType.LIMIT, price, position_action=PositionAction.CLOSE) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCompletedEvent)) + self.assertTrue(event.order_id is not None) + self.assertEqual(order_id, event.order_id) + self.assertEqual(event.base_asset_amount, amount) + print(event.order_id) + + def test_open_position_failure(self): + perfi = self.connector + # Since we don't have 1000000 xUSDC, this should trigger order failure + amount = Decimal("1000000") + price = Decimal("10") + order_id = perfi.sell(trading_pair, amount, OrderType.LIMIT, price, position_action=PositionAction.OPEN) + event = self.ev_loop.run_until_complete(self.event_logger.wait_for(MarketOrderFailureEvent)) + self.assertEqual(order_id, event.order_id) + + def test_filled_orders_recorded(self): + config_path = "test_config" + strategy_name = "test_strategy" + sql = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, db_path=self.db_path) + recorder = MarketsRecorder(sql, [self.connector], config_path, strategy_name) + recorder.start() + try: + self.connector._in_flight_orders.clear() + self.assertEqual(0, len(self.connector.tracking_states)) + + price: Decimal = Decimal("10") # quote_price * Decimal("0.8") + price = self.connector.quantize_order_price(trading_pair, price) + + amount: Decimal = Decimal("0.1") + amount = self.connector.quantize_order_amount(trading_pair, amount) + + sell_order_id = self.connector.sell(trading_pair, amount, OrderType.LIMIT, price, position_action=PositionAction.OPEN) + self.ev_loop.run_until_complete(self.event_logger.wait_for(SellOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + + price: Decimal = Decimal("10") # quote_price * Decimal("0.8") + price = self.connector.quantize_order_price(trading_pair, price) + + buy_order_id = self.connector.buy(trading_pair, amount, OrderType.LIMIT, price, position_action=PositionAction.CLOSE) + self.ev_loop.run_until_complete(self.event_logger.wait_for(BuyOrderCompletedEvent)) + self.ev_loop.run_until_complete(asyncio.sleep(1)) + + # Query the persisted trade logs + trade_fills: List[TradeFill] = recorder.get_trades_for_config(config_path) + # self.assertGreaterEqual(len(trade_fills), 2) + fills: List[TradeFill] = [t for t in trade_fills if t.trade_type == "SELL"] + self.assertGreaterEqual(len(fills), 1) + self.assertEqual(amount, Decimal(str(fills[0].amount))) + # self.assertEqual(price, Decimal(str(fills[0].price))) + self.assertEqual(base, fills[0].base_asset) + self.assertEqual(quote, fills[0].quote_asset) + self.assertEqual(sell_order_id, fills[0].order_id) + self.assertEqual(trading_pair, fills[0].symbol) + fills: List[TradeFill] = [t for t in trade_fills if t.trade_type == "BUY"] + self.assertGreaterEqual(len(fills), 1) + self.assertEqual(amount, Decimal(str(fills[0].amount))) + # self.assertEqual(price, Decimal(str(fills[0].price))) + self.assertEqual(base, fills[0].base_asset) + self.assertEqual(quote, fills[0].quote_asset) + self.assertEqual(buy_order_id, fills[0].order_id) + self.assertEqual(trading_pair, fills[0].symbol) + + finally: + recorder.stop() + os.unlink(self.db_path)