Merge pull request #1462 from bitshares/direct_index

Add new direct index type

libraries/app/database_api.cpp

@@ -547,6 +547,10 @@ vector<vector<account_id_type>> database_api::get_key_references( vector<public_
  */
 vector<vector<account_id_type>> database_api_impl::get_key_references( vector<public_key_type> keys )const
 {
+   const auto& idx = _db.get_index_type<account_index>();
+   const auto& aidx = dynamic_cast<const base_primary_index&>(idx);
+   const auto& refs = aidx.get_secondary_index<graphene::chain::account_member_index>();
+
    vector< vector<account_id_type> > final_result;
    final_result.reserve(keys.size());
 
@@ -566,10 +570,6 @@ vector<vector<account_id_type>> database_api_impl::get_key_references( vector<pu
       subscribe_to_item( a4 );
       subscribe_to_item( a5 );
 
-      const auto& idx = _db.get_index_type<account_index>();
-      const auto& aidx = dynamic_cast<const primary_index<account_index>&>(idx);
-      const auto& refs = aidx.get_secondary_index<graphene::chain::account_member_index>();
-      auto itr = refs.account_to_key_memberships.find(key);
       vector<account_id_type> result;
 
       for( auto& a : {a1,a2,a3,a4,a5} )
@@ -585,6 +585,7 @@ vector<vector<account_id_type>> database_api_impl::get_key_references( vector<pu
           }
       }
 
+      auto itr = refs.account_to_key_memberships.find(key);
       if( itr != refs.account_to_key_memberships.end() )
       {
          result.reserve( result.size() + itr->second.size() );
@@ -620,7 +621,7 @@ bool database_api_impl::is_public_key_registered(string public_key) const
         return false;
     }
     const auto& idx = _db.get_index_type<account_index>();
-    const auto& aidx = dynamic_cast<const primary_index<account_index>&>(idx);
+    const auto& aidx = dynamic_cast<const base_primary_index&>(idx);
     const auto& refs = aidx.get_secondary_index<graphene::chain::account_member_index>();
     auto itr = refs.account_to_key_memberships.find(key);
     bool is_known = itr != refs.account_to_key_memberships.end();
@@ -755,6 +756,10 @@ std::map<string,full_account> database_api::get_full_accounts( const vector<stri
 
 std::map<std::string, full_account> database_api_impl::get_full_accounts( const vector<std::string>& names_or_ids, bool subscribe)
 {
+   const auto& proposal_idx = _db.get_index_type<proposal_index>();
+   const auto& pidx = dynamic_cast<const base_primary_index&>(proposal_idx);
+   const auto& proposals_by_account = pidx.get_secondary_index<graphene::chain::required_approval_index>();
+
    std::map<std::string, full_account> results;
 
    for (const std::string& account_name_or_id : names_or_ids)
@@ -784,9 +789,6 @@ std::map<std::string, full_account> database_api_impl::get_full_accounts( const
          acnt.cashback_balance = account->cashback_balance(_db);
       }
       // Add the account's proposals
-      const auto& proposal_idx = _db.get_index_type<proposal_index>();
-      const auto& pidx = dynamic_cast<const primary_index<proposal_index>&>(proposal_idx);
-      const auto& proposals_by_account = pidx.get_secondary_index<graphene::chain::required_approval_index>();
       auto  required_approvals_itr = proposals_by_account._account_to_proposals.find( account->id );
       if( required_approvals_itr != proposals_by_account._account_to_proposals.end() )
       {
@@ -797,11 +799,9 @@ std::map<std::string, full_account> database_api_impl::get_full_accounts( const
 
 
       // Add the account's balances
-      auto balance_range = _db.get_index_type<account_balance_index>().indices().get<by_account_asset>().equal_range(boost::make_tuple(account->id));
-      std::for_each(balance_range.first, balance_range.second,
-                    [&acnt](const account_balance_object& balance) {
-                       acnt.balances.emplace_back(balance);
-                    });
+      const auto& balances = _db.get_index_type< primary_index< account_balance_index > >().get_secondary_index< balances_by_account_index >().get_account_balances( account->id );
+      for( const auto balance : balances )
+         acnt.balances.emplace_back( *balance.second );
 
       // Add the account's vesting balances
       auto vesting_range = _db.get_index_type<vesting_balance_index>().indices().get<by_account>().equal_range(account->id);
@@ -869,7 +869,7 @@ vector<account_id_type> database_api::get_account_references( const std::string
 vector<account_id_type> database_api_impl::get_account_references( const std::string account_id_or_name )const
 {
    const auto& idx = _db.get_index_type<account_index>();
-   const auto& aidx = dynamic_cast<const primary_index<account_index>&>(idx);
+   const auto& aidx = dynamic_cast<const base_primary_index&>(idx);
    const auto& refs = aidx.get_secondary_index<graphene::chain::account_member_index>();
    const account_id_type account_id = get_account_from_string(account_id_or_name)->id;
    auto itr = refs.account_to_account_memberships.find(account_id);
@@ -953,10 +953,10 @@ vector<asset> database_api_impl::get_account_balances(const std::string& account
    if (assets.empty())
    {
       // if the caller passes in an empty list of assets, return balances for all assets the account owns
-      const account_balance_index& balance_index = _db.get_index_type<account_balance_index>();
-      auto range = balance_index.indices().get<by_account_asset>().equal_range(boost::make_tuple(acnt));
-      for (const account_balance_object& balance : boost::make_iterator_range(range.first, range.second))
-         result.push_back(asset(balance.get_balance()));
+      const auto& balance_index = _db.get_index_type< primary_index< account_balance_index > >();
+      const auto& balances = balance_index.get_secondary_index< balances_by_account_index >().get_account_balances( acnt );
+      for( const auto balance : balances )
+         result.push_back( balance.second->get_balance() );
    }
    else
    {

libraries/chain/account_object.cpp

@@ -269,4 +269,54 @@ void account_referrer_index::object_modified( const object& after  )
 {
 }
 
+const uint8_t  balances_by_account_index::bits = 20;
+const uint64_t balances_by_account_index::mask = (1ULL << balances_by_account_index::bits) - 1;
+
+void balances_by_account_index::object_inserted( const object& obj )
+{
+   const auto& abo = dynamic_cast< const account_balance_object& >( obj );
+   while( balances.size() < (abo.owner.instance.value >> bits) + 1 )
+   {
+      balances.reserve( (abo.owner.instance.value >> bits) + 1 );
+      balances.resize( balances.size() + 1 );
+      balances.back().resize( 1ULL << bits );
+   }
+   balances[abo.owner.instance.value >> bits][abo.owner.instance.value & mask][abo.asset_type] = &abo;
+}
+
+void balances_by_account_index::object_removed( const object& obj )
+{
+   const auto& abo = dynamic_cast< const account_balance_object& >( obj );
+   if( balances.size() < (abo.owner.instance.value >> bits) + 1 ) return;
+   balances[abo.owner.instance.value >> bits][abo.owner.instance.value & mask].erase( abo.asset_type );
+}
+
+void balances_by_account_index::about_to_modify( const object& before )
+{
+   ids_being_modified.emplace( before.id );
+}
+
+void balances_by_account_index::object_modified( const object& after  )
+{
+   FC_ASSERT( ids_being_modified.top() == after.id, "Modification of ID is not supported!");
+   ids_being_modified.pop();
+}
+
+const map< asset_id_type, const account_balance_object* >& balances_by_account_index::get_account_balances( const account_id_type& acct )const
+{
+   static const map< asset_id_type, const account_balance_object* > _empty;
+
+   if( balances.size() < (acct.instance.value >> bits) + 1 ) return _empty;
+   return balances[acct.instance.value >> bits][acct.instance.value & mask];
+}
+
+const account_balance_object* balances_by_account_index::get_account_balance( const account_id_type& acct, const asset_id_type& asset )const
+{
+   if( balances.size() < (acct.instance.value >> bits) + 1 ) return nullptr;
+   const auto& mine = balances[acct.instance.value >> bits][acct.instance.value & mask];
+   const auto itr = mine.find( asset );
+   if( mine.end() == itr ) return nullptr;
+   return itr->second;
+}
+
 } } // graphene::chain

libraries/chain/db_balance.cpp

@@ -33,11 +33,11 @@ namespace graphene { namespace chain {
 
 asset database::get_balance(account_id_type owner, asset_id_type asset_id) const
 {
-   auto& index = get_index_type<account_balance_index>().indices().get<by_account_asset>();
-   auto itr = index.find(boost::make_tuple(owner, asset_id));
-   if( itr == index.end() )
+   auto& index = get_index_type< primary_index< account_balance_index > >().get_secondary_index<balances_by_account_index>();
+   auto abo = index.get_account_balance( owner, asset_id );
+   if( !abo )
       return asset(0, asset_id);
-   return itr->get_balance();
+   return abo->get_balance();
 }
 
 asset database::get_balance(const account_object& owner, const asset_object& asset_obj) const
@@ -55,9 +55,9 @@ void database::adjust_balance(account_id_type account, asset delta )
    if( delta.amount == 0 )
       return;
 
-   auto& index = get_index_type<account_balance_index>().indices().get<by_account_asset>();
-   auto itr = index.find(boost::make_tuple(account, delta.asset_id));
-   if(itr == index.end())
+   auto& index = get_index_type< primary_index< account_balance_index > >().get_secondary_index<balances_by_account_index>();
+   auto abo = index.get_account_balance( account, delta.asset_id );
+   if( !abo )
    {
       FC_ASSERT( delta.amount > 0, "Insufficient Balance: ${a}'s balance of ${b} is less than required ${r}", 
                  ("a",account(*this).name)
@@ -72,8 +72,9 @@ void database::adjust_balance(account_id_type account, asset delta )
       });
    } else {
       if( delta.amount < 0 )
-         FC_ASSERT( itr->get_balance() >= -delta, "Insufficient Balance: ${a}'s balance of ${b} is less than required ${r}", ("a",account(*this).name)("b",to_pretty_string(itr->get_balance()))("r",to_pretty_string(-delta)));
-      modify(*itr, [delta](account_balance_object& b) {
+         FC_ASSERT( abo->get_balance() >= -delta, "Insufficient Balance: ${a}'s balance of ${b} is less than required ${r}",
+                    ("a",account(*this).name)("b",to_pretty_string(abo->get_balance()))("r",to_pretty_string(-delta)));
+      modify(*abo, [delta](account_balance_object& b) {
          b.adjust_balance(delta);
       });
    }

libraries/chain/db_block.cpp

@@ -681,14 +681,6 @@ processed_transaction database::_apply_transaction(const signed_transaction& trx
    }
    ptrx.operation_results = std::move(eval_state.operation_results);
 
-   if( head_block_time() < HARDFORK_CORE_1040_TIME ) // TODO totally remove this code block after hard fork
-   {
-      //Make sure the temp account has no non-zero balances
-      const auto& index = get_index_type<account_balance_index>().indices().get<by_account_asset>();
-      auto range = index.equal_range( boost::make_tuple( GRAPHENE_TEMP_ACCOUNT ) );
-      std::for_each(range.first, range.second, [](const account_balance_object& b) { FC_ASSERT(b.balance == 0); });
-   }
-
    return ptrx;
 } FC_CAPTURE_AND_RETHROW( (trx) ) }
 

libraries/chain/db_init.cpp

@@ -182,15 +182,15 @@ void database::initialize_indexes()
    _undo_db.set_max_size( GRAPHENE_MIN_UNDO_HISTORY );
 
    //Protocol object indexes
-   add_index< primary_index<asset_index> >();
+   add_index< primary_index<asset_index, 13> >(); // 8192 assets per chunk
    add_index< primary_index<force_settlement_index> >();
 
-   auto acnt_index = add_index< primary_index<account_index> >();
+   auto acnt_index = add_index< primary_index<account_index, 20> >(); // ~1 million accounts per chunk
    acnt_index->add_secondary_index<account_member_index>();
    acnt_index->add_secondary_index<account_referrer_index>();
 
-   add_index< primary_index<committee_member_index> >();
-   add_index< primary_index<witness_index> >();
+   add_index< primary_index<committee_member_index, 8> >(); // 256 members per chunk
+   add_index< primary_index<witness_index, 10> >(); // 1024 witnesses per chunk
    add_index< primary_index<limit_order_index > >();
    add_index< primary_index<call_order_index > >();
 
@@ -205,11 +205,14 @@ void database::initialize_indexes()
 
    //Implementation object indexes
    add_index< primary_index<transaction_index                             > >();
-   add_index< primary_index<account_balance_index                         > >();
-   add_index< primary_index<asset_bitasset_data_index                     > >();
+
+   auto bal_idx = add_index< primary_index<account_balance_index          > >();
+   bal_idx->add_secondary_index<balances_by_account_index>();
+
+   add_index< primary_index<asset_bitasset_data_index,                 13 > >(); // 8192
    add_index< primary_index<simple_index<global_property_object          >> >();
    add_index< primary_index<simple_index<dynamic_global_property_object  >> >();
-   add_index< primary_index<account_stats_index                           > >();
+   add_index< primary_index<account_stats_index,                       20 > >(); // 1 Mi
    add_index< primary_index<simple_index<asset_dynamic_data_object       >> >();
    add_index< primary_index<simple_index<block_summary_object            >> >();
    add_index< primary_index<simple_index<chain_property_object          > > >();

libraries/chain/db_maint.cpp

@@ -693,32 +693,26 @@ void distribute_fba_balances( database& db )
 void create_buyback_orders( database& db )
 {
    const auto& bbo_idx = db.get_index_type< buyback_index >().indices().get<by_id>();
-   const auto& bal_idx = db.get_index_type< account_balance_index >().indices().get< by_account_asset >();
+   const auto& bal_idx = db.get_index_type< primary_index< account_balance_index > >().get_secondary_index< balances_by_account_index >();
 
    for( const buyback_object& bbo : bbo_idx )
    {
       const asset_object& asset_to_buy = bbo.asset_to_buy(db);
       assert( asset_to_buy.buyback_account.valid() );
 
       const account_object& buyback_account = (*(asset_to_buy.buyback_account))(db);
-      asset_id_type next_asset = asset_id_type();
 
       if( !buyback_account.allowed_assets.valid() )
       {
          wlog( "skipping buyback account ${b} at block ${n} because allowed_assets does not exist", ("b", buyback_account)("n", db.head_block_num()) );
          continue;
       }
 
-      while( true )
+      for( const auto& entry : bal_idx.get_account_balances( buyback_account.id ) )
       {
-         auto it = bal_idx.lower_bound( boost::make_tuple( buyback_account.id, next_asset ) );
-         if( it == bal_idx.end() )
-            break;
-         if( it->owner != buyback_account.id )
-            break;
+         const auto* it = entry.second;
          asset_id_type asset_to_sell = it->asset_type;
          share_type amount_to_sell = it->balance;
-         next_asset = asset_to_sell + 1;
          if( asset_to_sell == asset_to_buy.id )
             continue;
          if( amount_to_sell == 0 )

libraries/chain/include/graphene/chain/account_object.hpp

@@ -335,7 +335,30 @@ namespace graphene { namespace chain {
          map< account_id_type, set<account_id_type> > referred_by;
    };
 
-   struct by_account_asset;
+   /**
+    *  @brief This secondary index will allow fast access to the balance objects
+    *         that belonging to an account.
+    */
+   class balances_by_account_index : public secondary_index
+   {
+      public:
+         virtual void object_inserted( const object& obj ) override;
+         virtual void object_removed( const object& obj ) override;
+         virtual void about_to_modify( const object& before ) override;
+         virtual void object_modified( const object& after  ) override;
+
+         const map< asset_id_type, const account_balance_object* >& get_account_balances( const account_id_type& acct )const;
+         const account_balance_object* get_account_balance( const account_id_type& acct, const asset_id_type& asset )const;
+
+      private:
+         static const uint8_t  bits;
+         static const uint64_t mask;
+
+         /** Maps each account to its balance objects */
+         vector< vector< map< asset_id_type, const account_balance_object* > > > balances;
+         std::stack< object_id_type > ids_being_modified;
+   };
+
    struct by_asset_balance;
    struct by_maintenance_flag;
    /**
@@ -347,13 +370,6 @@ namespace graphene { namespace chain {
          ordered_unique< tag<by_id>, member< object, object_id_type, &object::id > >,
          ordered_non_unique< tag<by_maintenance_flag>,
                              member< account_balance_object, bool, &account_balance_object::maintenance_flag > >,
-         ordered_unique< tag<by_account_asset>,
-            composite_key<
-               account_balance_object,
-               member<account_balance_object, account_id_type, &account_balance_object::owner>,
-               member<account_balance_object, asset_id_type, &account_balance_object::asset_type>
-            >
-         >,
          ordered_unique< tag<by_asset_balance>,
             composite_key<
                account_balance_object,