Improve update_expired_feeds performance #1093

libraries/chain/asset_evaluator.cpp

@@ -319,6 +319,20 @@ void_result asset_update_evaluator::do_apply(const asset_update_operation& o)
          d.cancel_settle_order(*itr);
    }
 
+   // For market-issued assets, if core change rate changed, update flag in bitasset data
+   if( asset_to_update->is_market_issued()
+          && asset_to_update->options.core_exchange_rate != o.new_options.core_exchange_rate )
+   {
+      const auto& bitasset = asset_to_update->bitasset_data(d);
+      if( !bitasset.asset_cer_updated )
+      {
+         d.modify( bitasset, [](asset_bitasset_data_object& b)
+         {
+            b.asset_cer_updated = true;
+         });
+      }
+   }
+
    d.modify(*asset_to_update, [&o](asset_object& a) {
       if( o.new_issuer )
          a.issuer = *o.new_issuer;

libraries/chain/asset_object.cpp

@@ -70,12 +70,15 @@ void graphene::chain::asset_bitasset_data_object::update_median_feeds(time_point
    if( current_feeds.size() < options.minimum_feeds )
    {
       //... don't calculate a median, and set a null feed
+      feed_cer_updated = false; // new median cer is null, won't update asset_object anyway, set to false for better performance
       current_feed_publication_time = current_time;
       current_feed = price_feed();
       return;
    }
    if( current_feeds.size() == 1 )
    {
+      if( current_feed.core_exchange_rate != current_feeds.front().get().core_exchange_rate )
+         feed_cer_updated = true;
       current_feed = std::move(current_feeds.front());
       return;
    }
@@ -94,6 +97,8 @@ void graphene::chain::asset_bitasset_data_object::update_median_feeds(time_point
 #undef CALCULATE_MEDIAN_VALUE
    // *** End Median Calculations ***
 
+   if( current_feed.core_exchange_rate != median_feed.core_exchange_rate )
+      feed_cer_updated = true;
    current_feed = median_feed;
 }
 

libraries/chain/db_block.cpp

@@ -503,12 +503,14 @@ void database::_apply_block( const signed_block& next_block )
 
    const witness_object& signing_witness = validate_block_header(skip, next_block);
    const auto& global_props = get_global_properties();
-   const auto& dynamic_global_props = get<dynamic_global_property_object>(dynamic_global_property_id_type());
+   const auto& dynamic_global_props = get_dynamic_global_properties();
    bool maint_needed = (dynamic_global_props.next_maintenance_time <= next_block.timestamp);
 
    _current_block_num    = next_block_num;
    _current_trx_in_block = 0;
 
+   _issue_453_affected_assets.clear();
+
    for( const auto& trx : next_block.transactions )
    {
       /* We do not need to push the undo state for each transaction
@@ -535,6 +537,7 @@ void database::_apply_block( const signed_block& next_block )
    clear_expired_proposals();
    clear_expired_orders();
    update_expired_feeds();
+   update_core_exchange_rates();
    update_withdraw_permissions();
 
    // n.b., update_maintenance_flag() happens this late

libraries/chain/db_market.cpp

@@ -918,19 +918,21 @@ bool database::fill_settle_order( const force_settlement_object& settle, const a
  *
  *  @return true if a margin call was executed.
  */
-bool database::check_call_orders(const asset_object& mia, bool enable_black_swan, bool for_new_limit_order )
+bool database::check_call_orders( const asset_object& mia, bool enable_black_swan, bool for_new_limit_order,
+                                  const asset_bitasset_data_object* bitasset_ptr )
 { try {
-    auto head_time = head_block_time();
-    auto maint_time = get_dynamic_global_properties().next_maintenance_time;
+    static const auto& dyn_prop = get_dynamic_global_properties();
+    auto maint_time = dyn_prop.next_maintenance_time;
     if( for_new_limit_order )
        FC_ASSERT( maint_time <= HARDFORK_CORE_625_TIME ); // `for_new_limit_order` is only true before HF 338 / 625
 
     if( !mia.is_market_issued() ) return false;
 
-    if( check_for_blackswan( mia, enable_black_swan ) ) 
+    const asset_bitasset_data_object& bitasset = ( bitasset_ptr ? *bitasset_ptr : mia.bitasset_data(*this) );
+
+    if( check_for_blackswan( mia, enable_black_swan, &bitasset ) )
        return false;
 
-    const asset_bitasset_data_object& bitasset = mia.bitasset_data(*this);
     if( bitasset.is_prediction_market ) return false;
     if( bitasset.current_feed.settlement_price.is_null() ) return false;
 
@@ -961,6 +963,10 @@ bool database::check_call_orders(const asset_object& mia, bool enable_black_swan
     bool filled_limit = false;
     bool margin_called = false;
 
+    auto head_time = head_block_time();
+    auto head_num = head_block_num();
+
+    bool before_hardfork_615 = ( head_time < HARDFORK_615_TIME );
     bool after_hardfork_436 = ( head_time > HARDFORK_436_TIME );
 
     bool before_core_hardfork_184 = ( maint_time <= HARDFORK_CORE_184_TIME ); // something-for-nothing
@@ -970,7 +976,7 @@ bool database::check_call_orders(const asset_object& mia, bool enable_black_swan
     bool before_core_hardfork_606 = ( maint_time <= HARDFORK_CORE_606_TIME ); // feed always trigger call
     bool before_core_hardfork_834 = ( maint_time <= HARDFORK_CORE_834_TIME ); // target collateral ratio option
 
-    while( !check_for_blackswan( mia, enable_black_swan ) && call_itr != call_end )
+    while( !check_for_blackswan( mia, enable_black_swan, &bitasset ) && call_itr != call_end )
     {
        bool  filled_call      = false;
        price match_price;
@@ -1000,7 +1006,7 @@ bool database::check_call_orders(const asset_object& mia, bool enable_black_swan
        if( usd_to_buy * match_price > call_itr->get_collateral() )
        {
           elog( "black swan detected on asset ${symbol} (${id}) at block ${b}",
-                ("id",mia.id)("symbol",mia.symbol)("b",head_block_num()) );
+                ("id",mia.id)("symbol",mia.symbol)("b",head_num) );
           edump((enable_black_swan));
           FC_ASSERT( enable_black_swan );
           globally_settle_asset(mia, bitasset.current_feed.settlement_price );
@@ -1027,9 +1033,9 @@ bool database::check_call_orders(const asset_object& mia, bool enable_black_swan
           if( order_receives.amount == 0 ) // TODO this should not happen. remove the warning after confirmed
           {
              if( before_core_hardfork_184 )
-                wlog( "Something for nothing issue (#184, variant D-1) occurred at block #${block}", ("block",head_block_num()) );
+                wlog( "Something for nothing issue (#184, variant D-1) occurred at block #${block}", ("block",head_num) );
              else
-                wlog( "Something for nothing issue (#184, variant D-2) occurred at block #${block}", ("block",head_block_num()) );
+                wlog( "Something for nothing issue (#184, variant D-2) occurred at block #${block}", ("block",head_num) );
           }
 
           if( before_core_hardfork_342 )
@@ -1052,7 +1058,7 @@ bool database::check_call_orders(const asset_object& mia, bool enable_black_swan
 
              // Be here, the limit order would be paying something for nothing
              if( order_receives.amount == 0 ) // TODO remove warning after hard fork core-342
-                wlog( "Something for nothing issue (#184, variant D) occurred at block #${block}", ("block",head_block_num()) );
+                wlog( "Something for nothing issue (#184, variant D) occurred at block #${block}", ("block",head_num) );
           }
           else
              order_receives = usd_to_buy.multiply_and_round_up( match_price ); // round up, in favor of limit order
@@ -1062,7 +1068,11 @@ bool database::check_call_orders(const asset_object& mia, bool enable_black_swan
           if( usd_to_buy == usd_for_sale )
              filled_limit = true;
           else if( filled_limit && maint_time <= HARDFORK_CORE_453_TIME ) // TODO remove warning after hard fork core-453
-             wlog( "Multiple limit match problem (issue 453) occurred at block #${block}", ("block",head_block_num()) );
+          {
+             wlog( "Multiple limit match problem (issue 453) occurred at block #${block}", ("block",head_num) );
+             if( before_hardfork_615 )
+                _issue_453_affected_assets.insert( bitasset.asset_id );
+          }
        }
 
        call_pays  = order_receives;

libraries/chain/db_update.cpp

@@ -109,6 +109,7 @@ void database::update_last_irreversible_block()
    const global_property_object& gpo = get_global_properties();
    const dynamic_global_property_object& dpo = get_dynamic_global_properties();
 
+   // TODO for better performance, move this to db_maint, because only need to do it once per maintenance interval
    vector< const witness_object* > wit_objs;
    wit_objs.reserve( gpo.active_witnesses.size() );
    for( const witness_id_type& wid : gpo.active_witnesses )
@@ -180,11 +181,12 @@ void database::clear_expired_proposals()
  *
  *  A black swan occurs if MAX(HB,SP) <= LC
  */
-bool database::check_for_blackswan( const asset_object& mia, bool enable_black_swan )
+bool database::check_for_blackswan( const asset_object& mia, bool enable_black_swan,
+                                    const asset_bitasset_data_object* bitasset_ptr )
 {
     if( !mia.is_market_issued() ) return false;
 
-    const asset_bitasset_data_object& bitasset = mia.bitasset_data(*this);
+    const asset_bitasset_data_object& bitasset = ( bitasset_ptr ? *bitasset_ptr : mia.bitasset_data(*this) );
     if( bitasset.has_settlement() ) return true; // already force settled
     auto settle_price = bitasset.current_feed.settlement_price;
     if( settle_price.is_null() ) return false; // no feed
@@ -201,7 +203,8 @@ bool database::check_for_blackswan( const asset_object& mia, bool enable_black_s
 
     price highest = settle_price;
 
-    auto maint_time = get_dynamic_global_properties().next_maintenance_time;
+    static const auto& dyn_prop = get_dynamic_global_properties();
+    auto maint_time = dyn_prop.next_maintenance_time;
     if( maint_time > HARDFORK_CORE_338_TIME )
        // due to #338, we won't check for black swan on incoming limit order, so need to check with MSSP here
        highest = bitasset.current_feed.max_short_squeeze_price();
@@ -455,40 +458,84 @@ void database::clear_expired_orders()
 
 void database::update_expired_feeds()
 {
-   const auto head_num = head_block_num();
    const auto head_time = head_block_time();
-   bool before_hf_615 = ( head_time < HARDFORK_615_TIME );
-   auto& asset_idx = get_index_type<asset_index>().indices().get<by_type>();
-   auto itr = asset_idx.lower_bound( true /** market issued */ );
-   while( itr != asset_idx.end() )
+   bool after_hardfork_615 = ( head_time >= HARDFORK_615_TIME );
+
+   const auto& idx = get_index_type<asset_bitasset_data_index>().indices().get<by_feed_expiration>();
+   auto itr = idx.begin();
+   while( itr != idx.end() && itr->feed_is_expired( head_time ) )
    {
-      const asset_object& a = *itr;
-      ++itr;
-      assert( a.is_market_issued() );
-
-      const asset_bitasset_data_object& b = a.bitasset_data(*this);
-      bool feed_is_expired;
-      if( before_hf_615 )
-         feed_is_expired = b.feed_is_expired_before_hardfork_615( head_time );
-      else
-         feed_is_expired = b.feed_is_expired( head_time );
-      if( feed_is_expired )
+      const asset_bitasset_data_object& b = *itr;
+      ++itr; // not always process begin() because old code skipped updating some assets before hf 615
+      bool update_cer = false; // for better performance, to only update bitasset once, also check CER in this function
+      const asset_object* asset_ptr = nullptr;
+      // update feeds, check margin calls
+      if( after_hardfork_615 || b.feed_is_expired_before_hardfork_615( head_time ) )
       {
-         modify(b, [head_time](asset_bitasset_data_object& a) {
-            a.update_median_feeds(head_time);
+         auto old_median_feed = b.current_feed;
+         modify( b, [head_time,&update_cer]( asset_bitasset_data_object& abdo )
+         {
+            abdo.update_median_feeds( head_time );
+            if( abdo.need_to_update_cer() )
+            {
+               update_cer = true;
+               abdo.asset_cer_updated = false;
+               abdo.feed_cer_updated = false;
+            }
          });
-         bool called_some = check_call_orders(b.current_feed.settlement_price.base.asset_id(*this));
-         if( called_some && before_hf_615 )
+         if( !b.current_feed.settlement_price.is_null() && !( b.current_feed == old_median_feed ) ) // `==` check is safe here
          {
-            wlog( "Graphene issue #615: called some for asset ${a} on block #${b}, feed really expired: ${f}",
-                  ("a", a.symbol) ("b", head_num) ("f", b.feed_is_expired(head_time)) );
+            asset_ptr = &b.asset_id( *this );
+            check_call_orders( *asset_ptr, true, false, &b );
          }
       }
-      if( !b.current_feed.core_exchange_rate.is_null() &&
-          a.options.core_exchange_rate != b.current_feed.core_exchange_rate )
-         modify(a, [&b](asset_object& a) {
-            a.options.core_exchange_rate = b.current_feed.core_exchange_rate;
+      // update CER
+      if( update_cer )
+      {
+         if( !asset_ptr )
+            asset_ptr = &b.asset_id( *this );
+         if( asset_ptr->options.core_exchange_rate != b.current_feed.core_exchange_rate )
+         {
+            modify( *asset_ptr, [&b]( asset_object& ao )
+            {
+               ao.options.core_exchange_rate = b.current_feed.core_exchange_rate;
+            });
+         }
+      }
+   } // for each asset whose feed is expired
+
+   // process assets affected by bitshares-core issue 453 before hard fork 615
+   if( !after_hardfork_615 )
+   {
+      for( asset_id_type a : _issue_453_affected_assets )
+      {
+         check_call_orders( a(*this) );
+      }
+   }
+}
+
+void database::update_core_exchange_rates()
+{
+   const auto& idx = get_index_type<asset_bitasset_data_index>().indices().get<by_cer_update>();
+   if( idx.begin() != idx.end() )
+   {
+      for( auto itr = idx.rbegin(); itr->need_to_update_cer(); itr = idx.rbegin() )
+      {
+         const asset_bitasset_data_object& b = *itr;
+         const asset_object& a = b.asset_id( *this );
+         if( a.options.core_exchange_rate != b.current_feed.core_exchange_rate )
+         {
+            modify( a, [&b]( asset_object& ao )
+            {
+               ao.options.core_exchange_rate = b.current_feed.core_exchange_rate;
+            });
+         }
+         modify( b, []( asset_bitasset_data_object& abdo )
+         {
+            abdo.asset_cer_updated = false;
+            abdo.feed_cer_updated = false;
          });
+      }
    }
 }
 

libraries/chain/include/graphene/chain/asset_object.hpp

@@ -216,6 +216,18 @@ namespace graphene { namespace chain {
          share_type settlement_fund;
          ///@}
 
+         /// Track whether core_exchange_rate in corresponding asset_object has updated
+         bool asset_cer_updated = false;
+
+         /// Track whether core exchange rate in current feed has updated
+         bool feed_cer_updated = false;
+
+         /// Whether need to update core_exchange_rate in asset_object
+         bool need_to_update_cer() const
+         {
+            return ( ( feed_cer_updated || asset_cer_updated ) && !current_feed.core_exchange_rate.is_null() );
+         }
+
          /// The time when @ref current_feed would expire
          time_point_sec feed_expiration_time()const
          {
@@ -243,11 +255,23 @@ namespace graphene { namespace chain {
    };
 
    struct by_short_backing_asset;
+   struct by_feed_expiration;
+   struct by_cer_update;
+
    typedef multi_index_container<
       asset_bitasset_data_object,
       indexed_by<
          ordered_unique< tag<by_id>, member< object, object_id_type, &object::id > >,
-         ordered_non_unique< tag<by_short_backing_asset>, bitasset_short_backing_asset_extractor >
+         ordered_non_unique< tag<by_short_backing_asset>, bitasset_short_backing_asset_extractor >,
+         ordered_unique< tag<by_feed_expiration>,
+            composite_key< asset_bitasset_data_object,
+               const_mem_fun< asset_bitasset_data_object, time_point_sec, &asset_bitasset_data_object::feed_expiration_time >,
+               member< asset_bitasset_data_object, asset_id_type, &asset_bitasset_data_object::asset_id >
+            >
+         >,
+         ordered_non_unique< tag<by_cer_update>,
+                             const_mem_fun< asset_bitasset_data_object, bool, &asset_bitasset_data_object::need_to_update_cer >
+         >
       >
    > asset_bitasset_data_object_multi_index_type;
    typedef generic_index<asset_bitasset_data_object, asset_bitasset_data_object_multi_index_type> asset_bitasset_data_index;
@@ -286,6 +310,8 @@ FC_REFLECT_DERIVED( graphene::chain::asset_bitasset_data_object, (graphene::db::
                     (is_prediction_market)
                     (settlement_price)
                     (settlement_fund)
+                    (asset_cer_updated)
+                    (feed_cer_updated)
                   )
 
 FC_REFLECT_DERIVED( graphene::chain::asset_object, (graphene::db::object),

libraries/chain/include/graphene/chain/config.hpp

@@ -121,7 +121,7 @@
 #define GRAPHENE_RECENTLY_MISSED_COUNT_INCREMENT             4
 #define GRAPHENE_RECENTLY_MISSED_COUNT_DECREMENT             3
 
-#define GRAPHENE_CURRENT_DB_VERSION                          "BTS2.16"
+#define GRAPHENE_CURRENT_DB_VERSION                          "BTS2.17"
 
 #define GRAPHENE_IRREVERSIBLE_THRESHOLD                      (70 * GRAPHENE_1_PERCENT)
 

libraries/chain/include/graphene/chain/database.hpp

@@ -387,7 +387,8 @@ namespace graphene { namespace chain {
          bool fill_settle_order( const force_settlement_object& settle, const asset& pays, const asset& receives,
                                  const price& fill_price, const bool is_maker );
 
-         bool check_call_orders( const asset_object& mia, bool enable_black_swan = true, bool for_new_limit_order = false );
+         bool check_call_orders( const asset_object& mia, bool enable_black_swan = true, bool for_new_limit_order = false,
+                                 const asset_bitasset_data_object* bitasset_ptr = nullptr );
 
          // helpers to fill_order
          void pay_order( const account_object& receiver, const asset& receives, const asset& pays );
@@ -456,9 +457,11 @@ namespace graphene { namespace chain {
          void clear_expired_proposals();
          void clear_expired_orders();
          void update_expired_feeds();
+         void update_core_exchange_rates();
          void update_maintenance_flag( bool new_maintenance_flag );
          void update_withdraw_permissions();
-         bool check_for_blackswan( const asset_object& mia, bool enable_black_swan = true );
+         bool check_for_blackswan( const asset_object& mia, bool enable_black_swan = true,
+                                   const asset_bitasset_data_object* bitasset_ptr = nullptr );
 
          ///Steps performed only at maintenance intervals
          ///@{
@@ -524,6 +527,9 @@ namespace graphene { namespace chain {
           * database::close() has not been called, or failed during execution.
           */
          bool                              _opened = false;
+
+         /// Tracks assets affected by bitshares-core issue #453 before hard fork #615 in one block
+         flat_set<asset_id_type>           _issue_453_affected_assets;
    };
 
    namespace detail