Market engine improvements.

For:
* #338 Margin call order fills at price of matching limit_order
* #343 Inconsistent sorting of call orders between matching against a limit order and a force settle order
* #453 Multiple limit order and call order matching issue
* #606 Undercollateralized short positions should be called regardless of asks

libraries/chain/db_market.cpp

@@ -364,6 +364,154 @@ bool database::apply_order(const limit_order_object& new_order_object, bool allo
    return maybe_cull_small_order( *this, *updated_order_object );
 }
 
+bool database::apply_order_hf_201803(const limit_order_object& new_order_object, bool allow_black_swan)
+{
+   auto order_id = new_order_object.id;
+   asset_id_type sell_asset_id = new_order_object.sell_asset_id();
+   asset_id_type recv_asset_id = new_order_object.receive_asset_id();
+
+   // We only need to check if the new order will match with others if it is at the front of the book
+   const auto& limit_price_idx = get_index_type<limit_order_index>().indices().get<by_price>();
+   auto limit_itr = limit_price_idx.lower_bound( boost::make_tuple( new_order_object.sell_price, order_id ) );
+   if( limit_itr != limit_price_idx.begin() )
+   {
+      --limit_itr;
+      if( limit_itr->sell_asset_id() == sell_asset_id && limit_itr->receive_asset_id() == recv_asset_id )
+         return false;
+   }
+
+   // Order matching should be in favor of the taker.
+   // When a new limit order is created, e.g. an ask, need to check if it will match the highest bid.
+   // We were checking call orders first. However, due to MSSR (maximum_short_squeeze_ratio),
+   // effective price of call orders may be lower than limit orders, so we should also check limit orders here.
+
+   // Question: will a new limit order trigger a black swan event?
+   //
+   // 1. as of writing, it's possible due to the call-order-and-limit-order overlapping issue:
+   //       https://github.com/bitshares/bitshares-core/issues/606 .
+   //    when it happens, a call order can be very big but don't match with the opposite,
+   //    even when price feed is too far away, further than swan price,
+   //    if the new limit order is in the same direction with the call orders, it can eat up all the opposite,
+   //    then the call order will lose support and trigger a black swan event.
+   // 2. after issue 606 is fixed, there will be no limit order on the opposite side "supporting" the call order,
+   //    so a new order in the same direction with the call order won't trigger a black swan event.
+   // 3. calling is one direction. if the new limit order is on the opposite direction,
+   //    no matter if matches with the call, it won't trigger a black swan event.
+   //
+   // Since it won't trigger a black swan, no need to check here.
+
+   // currently we don't do cross-market (triangle) matching.
+   // the limit order will only match with a call order if meet all of these:
+   // 1. it's buying collateral, which means sell_asset is the MIA, receive_asset is the backing asset.
+   // 2. sell_asset is not a prediction market
+   // 3. sell_asset is not globally settled
+   // 4. sell_asset has a valid price feed
+   // 5. the call order doesn't have enough collateral
+   // 6. the limit order provided a good price
+   bool to_check_call_orders = false;
+   const asset_object& sell_asset = sell_asset_id( *this );
+   //const asset_object& recv_asset = recv_asset_id( *this );
+   const asset_bitasset_data_object* sell_abd = nullptr;
+   if( sell_asset.is_market_issued() )
+   {
+      sell_abd = &sell_asset.bitasset_data( *this );
+      if( sell_abd->options.short_backing_asset == recv_asset_id
+          && !sell_abd->is_prediction_market
+          && !sell_abd->has_settlement()
+          && !sell_abd->current_feed.settlement_price.is_null() )
+      {
+         to_check_call_orders = true;
+      }
+   }
+
+   // this is the opposite side
+   auto max_price = ~new_order_object.sell_price;
+   limit_itr = limit_price_idx.lower_bound( max_price.max() );
+   auto limit_end = limit_price_idx.upper_bound( max_price );
+   bool to_check_limit_orders = (limit_itr != limit_end);
+
+   if( to_check_call_orders )
+   {
+      // check if there are margin calls
+      const auto& call_price_idx = get_index_type<call_order_index>().indices().get<by_price>();
+      auto call_min = price::min( recv_asset_id, sell_asset_id );
+      price min_call_price = sell_abd->current_feed.max_short_squeeze_price();
+      while( true )
+      {
+         auto call_itr = call_price_idx.lower_bound( call_min );
+         // there is this new limit order means there are short positions, so call_itr might be valid
+         const call_order_object& call_order = *call_itr;
+         price call_order_price = ~call_order.call_price;
+         if( call_order_price >= sell_abd->current_feed.settlement_price ) // has enough collateral
+            to_check_call_orders = false;
+         else
+         {
+            if( call_order_price < min_call_price ) // feed protected https://github.com/cryptonomex/graphene/issues/436
+               call_order_price = min_call_price;
+            if( call_order_price > new_order_object.sell_price ) // new limit order is too far away, can't match
+               to_check_call_orders = false;
+         }
+
+         if( !to_check_call_orders ) // finished checking call orders
+            break;
+
+         if( to_check_limit_orders ) // need to check both calls and limits
+         {
+            // fill as many limit orders as possible
+            bool finished = false;
+            while( !finished && limit_itr != limit_end && call_order_price > ~limit_itr->sell_price )
+            {
+               auto old_limit_itr = limit_itr;
+               ++limit_itr;
+               // match returns 2 when only the old order was fully filled. In this case, we keep matching; otherwise, we stop.
+               finished = ( match( new_order_object, *old_limit_itr, old_limit_itr->sell_price ) != 2 );
+            }
+            if( finished ) // if the new limit order is gone
+            {
+               to_check_limit_orders = false; // no need to check more limit orders as well
+               break;
+            }
+            if( limit_itr == limit_end ) // if no more limit order to check
+               to_check_limit_orders = false; // no need to check more limit orders as well
+         }
+
+         // now fill the call order
+         auto match_result = match( new_order_object, call_order, call_order_price );
+
+         if( match_result != 2 ) // if the new limit order is gone
+         {
+            to_check_limit_orders = false; // no need to check more limit orders as well
+            break;
+         }
+         // else the call order should be gone, do thing here
+
+      } // end while
+
+   } // end check call orders
+
+   if( to_check_limit_orders ) // still and only need to check limit orders
+   {
+      bool finished = false;
+      while( !finished && limit_itr != limit_end )
+      {
+         auto old_limit_itr = limit_itr;
+         ++limit_itr;
+         // match returns 2 when only the old order was fully filled. In this case, we keep matching; otherwise, we stop.
+         finished = ( match( new_order_object, *old_limit_itr, old_limit_itr->sell_price ) != 2 );
+      }
+   }
+
+   // TODO really need to find again?
+   const limit_order_object* updated_order_object = find< limit_order_object >( order_id );
+   if( updated_order_object == nullptr )
+      return true;
+
+   // before #555 we would have done maybe_cull_small_order() logic as a result of fill_order() being called by match() above
+   // however after #555 we need to get rid of small orders -- #555 hardfork defers logic that was done too eagerly before, and
+   // this is the point it's deferred to.
+   return maybe_cull_small_order( *this, *updated_order_object );
+}
+
 /**
  *  Matches the two orders,
  *
@@ -422,6 +570,48 @@ int database::match( const limit_order_object& bid, const limit_order_object& as
    return match<limit_order_object>( bid, ask, match_price );
 }
 
+int database::match( const limit_order_object& bid, const call_order_object& ask, const price& match_price )
+{
+   FC_ASSERT( bid.sell_asset_id() == ask.debt_type() );
+   FC_ASSERT( bid.receive_asset_id() == ask.collateral_type() );
+   FC_ASSERT( bid.for_sale > 0 && ask.debt > 0 && ask.collateral > 0 );
+
+   bool  filled_limit     = false;
+   bool  filled_call      = false;
+
+   asset usd_for_sale = bid.amount_for_sale();
+   asset usd_to_buy   = ask.get_debt();
+
+   asset call_pays, call_receives, order_pays, order_receives;
+   if( usd_to_buy >= usd_for_sale )
+   {  // fill limit order
+      call_receives   = usd_for_sale;
+      order_receives  = usd_for_sale * match_price; // round down here, in favor of call order
+      call_pays       = order_receives;
+      order_pays      = usd_for_sale;
+
+      filled_limit    = true;
+      filled_call     = ( usd_to_buy == usd_for_sale );
+   }
+   else
+   {  // fill call order
+      call_receives  = usd_to_buy;
+      order_receives = usd_to_buy * match_price; // round down here, in favor of call order
+      call_pays      = order_receives;
+      order_pays     = usd_to_buy;
+
+      filled_call    = true;
+   }
+
+   FC_ASSERT( filled_call || filled_limit );
+
+   int result = 0;
+   result |= fill_order( bid, order_pays, order_receives, false, match_price, false ); // the limit order is taker
+   result |= fill_order( ask, call_pays, call_receives, match_price, true ) << 1;      // the call order is maker
+   FC_ASSERT( result != 0 );
+   return result;
+}
+
 
 asset database::match( const call_order_object& call, 
                        const force_settlement_object& settle, 
@@ -518,6 +708,11 @@ bool database::fill_order( const call_order_object& order, const asset& pays, co
    FC_ASSERT( order.get_collateral().asset_id == pays.asset_id );
    FC_ASSERT( order.get_collateral() >= pays );
 
+   const asset_object& mia = receives.asset_id(*this);
+   FC_ASSERT( mia.is_market_issued() );
+
+   const auto& mia_bdo = mia.bitasset_data( *this );
+
    optional<asset> collateral_freed;
    modify( order, [&]( call_order_object& o ){
             o.debt       -= receives.amount;
@@ -527,9 +722,10 @@ bool database::fill_order( const call_order_object& order, const asset& pays, co
               collateral_freed = o.get_collateral();
               o.collateral = 0;
             }
+            else if( head_block_time() > HARDFORK_CORE_343_TIME )
+              o.call_price = price::call_price( o.get_debt(), o.get_collateral(),
+                                                mia_bdo.current_feed.maintenance_collateral_ratio );
        });
-   const asset_object& mia = receives.asset_id(*this);
-   assert( mia.is_market_issued() );
 
    const asset_dynamic_data_object& mia_ddo = mia.dynamic_asset_data_id(*this);
 
@@ -643,8 +839,10 @@ bool database::check_call_orders(const asset_object& mia, bool enable_black_swan
     bool filled_limit = false;
     bool margin_called = false;
 
+    auto head_time = head_block_time();
     while( !check_for_blackswan( mia, enable_black_swan ) && call_itr != call_end )
     {
+       bool  filled_limit_in_loop = false;
        bool  filled_call      = false;
        price match_price;
        asset usd_for_sale;
@@ -659,12 +857,12 @@ bool database::check_call_orders(const asset_object& mia, bool enable_black_swan
        match_price.validate();
 
        // would be margin called, but there is no matching order #436
-       bool feed_protected = ( bitasset.current_feed.settlement_price > ~call_itr->call_price );
-       if( feed_protected && (head_block_time() > HARDFORK_436_TIME) )
+       if( ( head_time > HARDFORK_436_TIME )
+             && ( bitasset.current_feed.settlement_price > ~call_itr->call_price ) )
           return margin_called;
 
        // would be margin called, but there is no matching order
-       if( match_price > ~call_itr->call_price )
+       if( head_time <= HARDFORK_CORE_606_TIME && match_price > ~call_itr->call_price )
           return margin_called;
 
        /*
@@ -702,6 +900,7 @@ bool database::check_call_orders(const asset_object& mia, bool enable_black_swan
           call_pays       = order_receives;
           order_pays      = usd_for_sale;
 
+          filled_limit_in_loop = true;
           filled_limit = true;
           filled_call           = (usd_to_buy == usd_for_sale);
        } else { // fill call
@@ -711,20 +910,31 @@ bool database::check_call_orders(const asset_object& mia, bool enable_black_swan
           order_pays     = usd_to_buy;
 
           filled_call    = true;
-          if( filled_limit )
+          if( filled_limit && head_time <= HARDFORK_CORE_453_TIME )
              wlog( "Multiple limit match problem (issue 338) occurred at block #${block}", ("block",head_block_num()) );
        }
 
        FC_ASSERT( filled_call || filled_limit );
+       FC_ASSERT( filled_call || filled_limit_in_loop );
 
        auto old_call_itr = call_itr;
-       if( filled_call ) ++call_itr;
+       if( filled_call && head_time <= HARDFORK_CORE_343_TIME )
+          ++call_itr;
        // when for_new_limit_order is true, the call order is maker, otherwise the call order is taker
        fill_order(*old_call_itr, call_pays, call_receives, match_price, for_new_limit_order );
+       if( head_time > HARDFORK_CORE_343_TIME )
+          call_itr = call_price_index.lower_bound( call_min );
 
        auto old_limit_itr = limit_itr;
        auto next_limit_itr = std::next( limit_itr );
-       if( filled_limit ) ++limit_itr;
+       if( head_time <= HARDFORK_CORE_453_TIME )
+       {
+          if( filled_limit ) ++limit_itr;
+       }
+       else
+       {
+          if( filled_limit_in_loop ) ++limit_itr;
+       }
        // when for_new_limit_order is true, the limit order is taker, otherwise the limit order is maker
        bool really_filled = fill_order(*old_limit_itr, order_pays, order_receives, true, match_price, !for_new_limit_order );
        if( !filled_limit && really_filled )

libraries/chain/db_update.cpp

@@ -235,6 +235,9 @@ bool database::check_for_blackswan( const asset_object& mia, bool enable_black_s
        highest = std::max( limit_itr->sell_price, settle_price );
     }
 
+    // FIXME
+    // possible BUG: position with lowest call_price doesn't always have least collateral
+    // related to https://github.com/bitshares/bitshares-core/issues/343
     auto least_collateral = call_itr->collateralization();
     if( ~least_collateral >= highest  ) 
     {

libraries/chain/hardfork.d/CORE_338.hf

@@ -0,0 +1,4 @@
+// bitshares-core issue #338 Fix "margin call order fills at price of matching limit_order"
+#ifndef HARDFORK_CORE_338_TIME
+#define HARDFORK_CORE_338_TIME (fc::time_point_sec( 1600000000 ))
+#endif

libraries/chain/hardfork.d/CORE_343.hf

@@ -0,0 +1,5 @@
+// bitshares-core issue #343
+// Fix "Inconsistent sorting of call orders between matching against a limit order and a force settle order"
+#ifndef HARDFORK_CORE_343_TIME
+#define HARDFORK_CORE_343_TIME (fc::time_point_sec( 1600000000 ))
+#endif

libraries/chain/hardfork.d/CORE_453.hf

@@ -0,0 +1,4 @@
+// bitshares-core issue #453 Fix "Multiple limit order and call order matching issue"
+#ifndef HARDFORK_CORE_453_TIME
+#define HARDFORK_CORE_453_TIME (fc::time_point_sec( 1600000000 ))
+#endif

libraries/chain/hardfork.d/CORE_606.hf

@@ -0,0 +1,4 @@
+// bitshares-core issue #606 Fix "Undercollateralized short positions should be called regardless of asks"
+#ifndef HARDFORK_CORE_606_TIME
+#define HARDFORK_CORE_606_TIME (fc::time_point_sec( 1600000000 ))
+#endif

libraries/chain/include/graphene/chain/database.hpp

@@ -352,6 +352,7 @@ namespace graphene { namespace chain {
           * already on the books.
           */
          bool apply_order(const limit_order_object& new_order_object, bool allow_black_swan = true);
+         bool apply_order_hf_201803(const limit_order_object& new_order_object, bool allow_black_swan = true);
 
          /**
           * Matches the two orders,
@@ -367,6 +368,7 @@ namespace graphene { namespace chain {
          template<typename OrderType>
          int match( const limit_order_object& bid, const OrderType& ask, const price& match_price );
          int match( const limit_order_object& bid, const limit_order_object& ask, const price& trade_price );
+         int match( const limit_order_object& bid, const call_order_object& ask, const price& trade_price );
          /// @return the amount of asset settled
          asset match(const call_order_object& call,
                    const force_settlement_object& settle,

libraries/chain/include/graphene/chain/market_object.hpp

@@ -64,6 +64,8 @@ class limit_order_object : public abstract_object<limit_order_object>
 
       asset amount_for_sale()const   { return asset( for_sale, sell_price.base.asset_id ); }
       asset amount_to_receive()const { return amount_for_sale() * sell_price; }
+      asset_id_type sell_asset_id()const    { return sell_price.base.asset_id;  }
+      asset_id_type receive_asset_id()const { return sell_price.quote.asset_id; }
 };
 
 struct by_id;
@@ -115,12 +117,13 @@ class call_order_object : public abstract_object<call_order_object>
       asset get_debt()const { return asset( debt, debt_type() ); }
       asset amount_to_receive()const { return get_debt(); }
       asset_id_type debt_type()const { return call_price.quote.asset_id; }
+      asset_id_type collateral_type()const { return call_price.base.asset_id; }
       price collateralization()const { return get_collateral() / get_debt(); }
 
       account_id_type  borrower;
       share_type       collateral;  ///< call_price.base.asset_id, access via get_collateral
-      share_type       debt;        ///< call_price.quote.asset_id, access via get_collateral
-      price            call_price;  ///< Debt / Collateral
+      share_type       debt;        ///< call_price.quote.asset_id, access via get_debt
+      price            call_price;  ///< Collateral / Debt
 
       pair<asset_id_type,asset_id_type> get_market()const
       {

libraries/chain/market_evaluator.cpp

@@ -111,7 +111,11 @@ object_id_type limit_order_create_evaluator::do_apply(const limit_order_create_o
        obj.deferred_paid_fee = _deferred_paid_fee;
    });
    limit_order_id_type order_id = new_order_object.id; // save this because we may remove the object by filling it
-   bool filled = db().apply_order(new_order_object);
+   bool filled;
+   if( db().head_block_time() <= HARDFORK_CORE_338_TIME )
+      filled = db().apply_order( new_order_object );
+   else
+      filled = db().apply_order_hf_201803( new_order_object );
 
    FC_ASSERT( !op.fill_or_kill || filled );