From ca884e2b9f079cece2912c722a5a60106fbd1d0c Mon Sep 17 00:00:00 2001
From: Haruka Ma <mrx@hcc.im>
Date: Tue, 30 Oct 2018 12:28:35 +0900
Subject: [PATCH 01/38] Change description of delayed_node option

---
 libraries/plugins/delayed_node/delayed_node_plugin.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/plugins/delayed_node/delayed_node_plugin.cpp b/libraries/plugins/delayed_node/delayed_node_plugin.cpp
index 24a46cc066..f780d8b496 100644
--- a/libraries/plugins/delayed_node/delayed_node_plugin.cpp
+++ b/libraries/plugins/delayed_node/delayed_node_plugin.cpp
@@ -58,7 +58,7 @@ delayed_node_plugin::~delayed_node_plugin()
 void delayed_node_plugin::plugin_set_program_options(bpo::options_description& cli, bpo::options_description& cfg)
 {
    cli.add_options()
-         ("trusted-node", boost::program_options::value<std::string>(), "RPC endpoint of a trusted validating node (required)")
+         ("trusted-node", boost::program_options::value<std::string>(), "RPC endpoint of a trusted validating node (required for delayed_node)")
          ;
    cfg.add(cli);
 }

From 15e903717fdad07eba083dd8505a2f2d5f7926ca Mon Sep 17 00:00:00 2001
From: Alfredo <oxarbitrage@gmail.com>
Date: Fri, 9 Nov 2018 16:20:59 -0300
Subject: [PATCH 02/38] refactor es_objects to use templates

---
 libraries/plugins/es_objects/es_objects.cpp   | 194 +++---------------
 .../graphene/es_objects/es_objects.hpp        | 157 +++++---------
 2 files changed, 82 insertions(+), 269 deletions(-)

diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp
index d0c35010ce..0cd4bc256d 100644
--- a/libraries/plugins/es_objects/es_objects.cpp
+++ b/libraries/plugins/es_objects/es_objects.cpp
@@ -30,10 +30,12 @@
 #include <graphene/chain/proposal_object.hpp>
 #include <graphene/chain/balance_object.hpp>
 #include <graphene/chain/market_object.hpp>
+#include <graphene/chain/asset_object.hpp>
+#include <graphene/chain/account_object.hpp>
+#include <boost/algorithm/string.hpp>
 
 #include <graphene/utilities/elasticsearch.hpp>
 
-
 namespace graphene { namespace es_objects {
 
 namespace detail
@@ -72,12 +74,8 @@ class es_objects_plugin_impl
       fc::time_point_sec block_time;
 
    private:
-      void prepare_proposal(const proposal_object& proposal_object);
-      void prepare_account(const account_object& account_object);
-      void prepare_asset(const asset_object& asset_object);
-      void prepare_balance(const account_balance_object& account_balance_object);
-      void prepare_limit(const limit_order_object& limit_object);
-      void prepare_bitasset(const asset_bitasset_data_object& bitasset_object);
+       template<typename T>
+       void prepareTemplate(T blockchain_object, string index_name);
 };
 
 bool es_objects_plugin_impl::index_database( const vector<object_id_type>& ids, std::string action)
@@ -102,7 +100,7 @@ bool es_objects_plugin_impl::index_database( const vector<object_id_type>& ids,
             if(action == "delete")
                remove_from_database(p->id, "proposal");
             else
-               prepare_proposal(*p);
+               prepareTemplate<proposal_object>(*p, "proposal");
          }
       }
       else if(value.is<account_object>() && _es_objects_accounts) {
@@ -112,7 +110,7 @@ bool es_objects_plugin_impl::index_database( const vector<object_id_type>& ids,
             if(action == "delete")
                remove_from_database(a->id, "account");
             else
-               prepare_account(*a);
+               prepareTemplate<account_object>(*a, "account");
          }
       }
       else if(value.is<asset_object>() && _es_objects_assets) {
@@ -122,7 +120,7 @@ bool es_objects_plugin_impl::index_database( const vector<object_id_type>& ids,
             if(action == "delete")
                remove_from_database(a->id, "asset");
             else
-               prepare_asset(*a);
+               prepareTemplate<asset_object>(*a, "asset");
          }
       }
       else if(value.is<account_balance_object>() && _es_objects_balances) {
@@ -132,7 +130,7 @@ bool es_objects_plugin_impl::index_database( const vector<object_id_type>& ids,
             if(action == "delete")
                remove_from_database(b->id, "balance");
             else
-               prepare_balance(*b);
+               prepareTemplate<account_balance_object>(*b, "balance");
          }
       }
       else if(value.is<limit_order_object>() && _es_objects_limit_orders) {
@@ -142,7 +140,7 @@ bool es_objects_plugin_impl::index_database( const vector<object_id_type>& ids,
             if(action == "delete")
                remove_from_database(l->id, "limitorder");
             else
-               prepare_limit(*l);
+               prepareTemplate<limit_order_object>(*l, "limitorder");
          }
       }
       else if(value.is<asset_bitasset_data_object>() && _es_objects_asset_bitasset) {
@@ -152,7 +150,7 @@ bool es_objects_plugin_impl::index_database( const vector<object_id_type>& ids,
             if(action == "delete")
                remove_from_database(ba->id, "bitasset");
             else
-               prepare_bitasset(*ba);
+               prepareTemplate<asset_bitasset_data_object>(*ba, "bitasset");
          }
       }
    }
@@ -190,181 +188,39 @@ void es_objects_plugin_impl::remove_from_database( object_id_type id, std::strin
    }
 }
 
-void es_objects_plugin_impl::prepare_proposal(const proposal_object& proposal_object)
-{
-   proposal_struct prop;
-   prop.object_id = proposal_object.id;
-   prop.block_time = block_time;
-   prop.block_number = block_number;
-   prop.expiration_time = proposal_object.expiration_time;
-   prop.review_period_time = proposal_object.review_period_time;
-   prop.proposed_transaction = fc::json::to_string(proposal_object.proposed_transaction);
-   prop.required_owner_approvals = fc::json::to_string(proposal_object.required_owner_approvals);
-   prop.available_owner_approvals = fc::json::to_string(proposal_object.available_owner_approvals);
-   prop.required_active_approvals = fc::json::to_string(proposal_object.required_active_approvals);
-   prop.available_key_approvals = fc::json::to_string(proposal_object.available_key_approvals);
-   prop.proposer = proposal_object.proposer;
-
-   std::string data = fc::json::to_string(prop);
-
-   fc::mutable_variant_object bulk_header;
-   bulk_header["_index"] = _es_objects_index_prefix + "proposal";
-   bulk_header["_type"] = "data";
-   if(_es_objects_keep_only_current)
-   {
-      bulk_header["_id"] = string(prop.object_id);
-   }
-
-   prepare = graphene::utilities::createBulk(bulk_header, std::move(data));
-   std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk));
-   prepare.clear();
-}
-
-void es_objects_plugin_impl::prepare_account(const account_object& account_object)
-{
-   account_struct acct;
-   acct.object_id = account_object.id;
-   acct.block_time = block_time;
-   acct.block_number = block_number;
-   acct.membership_expiration_date = account_object.membership_expiration_date;
-   acct.registrar = account_object.registrar;
-   acct.referrer = account_object.referrer;
-   acct.lifetime_referrer = account_object.lifetime_referrer;
-   acct.network_fee_percentage = account_object.network_fee_percentage;
-   acct.lifetime_referrer_fee_percentage = account_object.lifetime_referrer_fee_percentage;
-   acct.referrer_rewards_percentage = account_object.referrer_rewards_percentage;
-   acct.name = account_object.name;
-   acct.owner_account_auths = fc::json::to_string(account_object.owner.account_auths);
-   acct.owner_key_auths = fc::json::to_string(account_object.owner.key_auths);
-   acct.owner_address_auths = fc::json::to_string(account_object.owner.address_auths);
-   acct.active_account_auths = fc::json::to_string(account_object.active.account_auths);
-   acct.active_key_auths = fc::json::to_string(account_object.active.key_auths);
-   acct.active_address_auths = fc::json::to_string(account_object.active.address_auths);
-   acct.voting_account = account_object.options.voting_account;
-   acct.votes = fc::json::to_string(account_object.options.votes);
-
-   std::string data = fc::json::to_string(acct);
-
-   fc::mutable_variant_object bulk_header;
-   bulk_header["_index"] = _es_objects_index_prefix + "account";
-   bulk_header["_type"] = "data";
-   if(_es_objects_keep_only_current)
-   {
-      bulk_header["_id"] = string(acct.object_id);
-   }
-
-   prepare = graphene::utilities::createBulk(bulk_header, std::move(data));
-   std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk));
-   prepare.clear();
-}
-
-void es_objects_plugin_impl::prepare_asset(const asset_object& asset_object)
+template<typename T>
+void es_objects_plugin_impl::prepareTemplate(T blockchain_object, string index_name)
 {
-   asset_struct asset;
-   asset.object_id = asset_object.id;
-   asset.block_time = block_time;
-   asset.block_number = block_number;
-   asset.symbol = asset_object.symbol;
-   asset.issuer = asset_object.issuer;
-   asset.is_market_issued = asset_object.is_market_issued();
-   asset.dynamic_asset_data_id = asset_object.dynamic_asset_data_id;
-   asset.bitasset_data_id = asset_object.bitasset_data_id;
-
-   std::string data = fc::json::to_string(asset);
-
    fc::mutable_variant_object bulk_header;
-   bulk_header["_index"] = _es_objects_index_prefix + "asset";
+   bulk_header["_index"] = _es_objects_index_prefix + index_name;
    bulk_header["_type"] = "data";
    if(_es_objects_keep_only_current)
    {
-      bulk_header["_id"] = string(asset.object_id);
+      bulk_header["_id"] = string(blockchain_object.id);
    }
 
-   prepare = graphene::utilities::createBulk(bulk_header, std::move(data));
-   std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk));
-   prepare.clear();
-}
+   auto blockchain_object_string = fc::json::to_string(blockchain_object, fc::json::legacy_generator);
 
-void es_objects_plugin_impl::prepare_balance(const account_balance_object& account_balance_object)
-{
-   balance_struct balance;
-   balance.object_id = account_balance_object.id;
-   balance.block_time = block_time;
-   balance.block_number = block_number;
-   balance.owner = account_balance_object.owner;
-   balance.asset_type = account_balance_object.asset_type;
-   balance.balance = account_balance_object.balance;
-   balance.maintenance_flag = account_balance_object.maintenance_flag;
+   variant j = fc::json::from_string(blockchain_object_string);
 
-   std::string data = fc::json::to_string(balance);
+   fc::mutable_variant_object o;
 
-   fc::mutable_variant_object bulk_header;
-   bulk_header["_index"] = _es_objects_index_prefix + "balance";
-   bulk_header["_type"] = "data";
-   if(_es_objects_keep_only_current)
-   {
-      bulk_header["_id"] = string(balance.object_id);
-   }
+   adaptor_struct adaptor;
+   auto adapted = adaptor.adapt(j.get_object());
 
-   prepare = graphene::utilities::createBulk(bulk_header, std::move(data));
-   std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk));
-   prepare.clear();
-}
+   fc::from_variant(adapted, o, FC_PACK_MAX_DEPTH);
 
-void es_objects_plugin_impl::prepare_limit(const limit_order_object& limit_object)
-{
-   limit_order_struct limit;
-   limit.object_id = limit_object.id;
-   limit.block_time = block_time;
-   limit.block_number = block_number;
-   limit.expiration = limit_object.expiration;
-   limit.seller = limit_object.seller;
-   limit.for_sale = limit_object.for_sale;
-   limit.sell_price = limit_object.sell_price;
-   limit.deferred_fee = limit_object.deferred_fee;
-
-   std::string data = fc::json::to_string(limit);
+   o["object_id"] = string(blockchain_object.id);
+   o["block_time"] = block_time;
+   o["block_number"] = block_number;
 
-   fc::mutable_variant_object bulk_header;
-   bulk_header["_index"] = _es_objects_index_prefix + "limitorder";
-   bulk_header["_type"] = "data";
-   if(_es_objects_keep_only_current)
-   {
-      bulk_header["_id"] = string(limit.object_id);
-   }
+   string data = fc::json::to_string(o, fc::json::legacy_generator);
 
    prepare = graphene::utilities::createBulk(bulk_header, std::move(data));
    std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk));
    prepare.clear();
 }
 
-void es_objects_plugin_impl::prepare_bitasset(const asset_bitasset_data_object& bitasset_object)
-{
-   if(!bitasset_object.is_prediction_market) {
-
-      bitasset_struct bitasset;
-      bitasset.object_id = bitasset_object.id;
-      bitasset.block_time = block_time;
-      bitasset.block_number = block_number;
-      bitasset.current_feed = fc::json::to_string(bitasset_object.current_feed);
-      bitasset.current_feed_publication_time = bitasset_object.current_feed_publication_time;
-
-      std::string data = fc::json::to_string(bitasset);
-
-      fc::mutable_variant_object bulk_header;
-      bulk_header["_index"] = _es_objects_index_prefix + "bitasset";
-      bulk_header["_type"] = "data";
-      if(_es_objects_keep_only_current)
-      {
-         bulk_header["_id"] = string(bitasset.object_id);
-      }
-
-      prepare = graphene::utilities::createBulk(bulk_header, std::move(data));
-      std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk));
-      prepare.clear();
-   }
-}
-
 es_objects_plugin_impl::~es_objects_plugin_impl()
 {
    return;
diff --git a/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp b/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp
index 54ecbdc90d..a66e299551 100644
--- a/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp
+++ b/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp
@@ -30,7 +30,6 @@ namespace graphene { namespace es_objects {
 
 using namespace chain;
 
-
 namespace detail
 {
     class es_objects_plugin_impl;
@@ -54,105 +53,63 @@ class es_objects_plugin : public graphene::app::plugin
       std::unique_ptr<detail::es_objects_plugin_impl> my;
 };
 
-struct proposal_struct {
-   object_id_type object_id;
-   fc::time_point_sec block_time;
-   uint32_t block_number;
-   time_point_sec expiration_time;
-   optional<time_point_sec> review_period_time;
-   string proposed_transaction;
-   string required_active_approvals;
-   string available_active_approvals;
-   string required_owner_approvals;
-   string available_owner_approvals;
-   string available_key_approvals;
-   account_id_type proposer;
-};
-struct account_struct {
-   object_id_type object_id;
-   fc::time_point_sec block_time;
-   uint32_t block_number;
-   time_point_sec membership_expiration_date;
-   account_id_type registrar;
-   account_id_type referrer;
-   account_id_type lifetime_referrer;
-   uint16_t network_fee_percentage;
-   uint16_t lifetime_referrer_fee_percentage;
-   uint16_t referrer_rewards_percentage;
-   string name;
-   string owner_account_auths;
-   string owner_key_auths;
-   string owner_address_auths;
-   string active_account_auths;
-   string active_key_auths;
-   string active_address_auths;
-   account_id_type voting_account;
-   string votes;
-};
-struct asset_struct {
-   object_id_type object_id;
-   fc::time_point_sec block_time;
-   uint32_t block_number;
-   string symbol;
-   account_id_type issuer;
-   bool is_market_issued;
-   asset_dynamic_data_id_type dynamic_asset_data_id;
-   optional<asset_bitasset_data_id_type> bitasset_data_id;
-};
-struct balance_struct {
-   object_id_type object_id;
-   fc::time_point_sec block_time;
-   uint32_t block_number;
-   account_id_type owner;
-   asset_id_type asset_type;
-   share_type balance;
-   bool maintenance_flag;
-};
-struct limit_order_struct {
-   object_id_type object_id;
-   fc::time_point_sec block_time;
-   uint32_t block_number;
-   time_point_sec expiration;
-   account_id_type seller;
-   share_type for_sale;
-   price sell_price;
-   share_type deferred_fee;
-};
-struct bitasset_struct {
-   object_id_type object_id;
-   fc::time_point_sec block_time;
-   uint32_t block_number;
-   string current_feed;
-   time_point_sec current_feed_publication_time;
-   time_point_sec feed_expiration_time;
+struct adaptor_struct {
+   variant adapt(const variant_object &obj) {
+      fc::mutable_variant_object o(obj);
+      vector<string> keys_to_rename;
+      for (auto i = o.begin(); i != o.end(); ++i) {
+         auto &element = (*i).value();
+         if (element.is_object()) {
+            const string &name = (*i).key();
+            auto &vo = element.get_object();
+            if (vo.contains(name.c_str()))
+               keys_to_rename.emplace_back(name);
+            element = adapt(vo);
+         } else if (element.is_array())
+            adapt(element.get_array());
+      }
+      for (const auto &i : keys_to_rename) {
+         string new_name = i + "_";
+         o[new_name] = variant(o[i]);
+         o.erase(i);
+      }
+      if (o.find("owner") != o.end() && o["owner"].is_string())
+      {
+         o["owner_"] = o["owner"].as_string();
+         o.erase("owner");
+      }
+      if (o.find("active_special_authority") != o.end())
+      {
+         o["active_special_authority"] = fc::json::to_string(o["active_special_authority"]);
+      }
+      if (o.find("owner_special_authority") != o.end())
+      {
+         o["owner_special_authority"] = fc::json::to_string(o["owner_special_authority"]);
+      }
+      if (o.find("feeds") != o.end())
+      {
+         o["feeds"] = fc::json::to_string(o["feeds"]);
+      }
+      if (o.find("operations") != o.end())
+      {
+         o["operations"] = fc::json::to_string(o["operations"]);
+      }
+
+      variant v;
+      fc::to_variant(o, v, FC_PACK_MAX_DEPTH);
+      return v;
+   }
+
+   void adapt(fc::variants &v) {
+      for (auto &array_element : v) {
+         if (array_element.is_object())
+            array_element = adapt(array_element.get_object());
+         else if (array_element.is_array())
+            adapt(array_element.get_array());
+         else
+            array_element = array_element.as_string();
+      }
+   }
 };
 
 } } //graphene::es_objects
-
-FC_REFLECT(
-        graphene::es_objects::proposal_struct,
-        (object_id)(block_time)(block_number)(expiration_time)(review_period_time)(proposed_transaction)(required_active_approvals)
-        (available_active_approvals)(required_owner_approvals)(available_owner_approvals)(available_key_approvals)(proposer)
-)
-FC_REFLECT(
-        graphene::es_objects::account_struct,
-        (object_id)(block_time)(block_number)(membership_expiration_date)(registrar)(referrer)(lifetime_referrer)
-        (network_fee_percentage)(lifetime_referrer_fee_percentage)(referrer_rewards_percentage)(name)(owner_account_auths)
-        (owner_key_auths)(owner_address_auths)(active_account_auths)(active_key_auths)(active_address_auths)(voting_account)(votes)
-)
-FC_REFLECT(
-        graphene::es_objects::asset_struct,
-        (object_id)(block_time)(block_number)(symbol)(issuer)(is_market_issued)(dynamic_asset_data_id)(bitasset_data_id)
-)
-FC_REFLECT(
-        graphene::es_objects::balance_struct,
-        (object_id)(block_time)(block_number)(owner)(asset_type)(balance)(maintenance_flag)
-)
-FC_REFLECT(
-        graphene::es_objects::limit_order_struct,
-        (object_id)(block_time)(block_number)(expiration)(seller)(for_sale)(sell_price)(deferred_fee)
-)
-FC_REFLECT(
-        graphene::es_objects::bitasset_struct,
-        (object_id)(block_time)(block_number)(current_feed)(current_feed_publication_time)
-)
\ No newline at end of file

From 6225ee42f6ea847c7370f7e407c0f15c09ee464e Mon Sep 17 00:00:00 2001
From: Alfredo <oxarbitrage@gmail.com>
Date: Sat, 10 Nov 2018 16:18:24 -0300
Subject: [PATCH 03/38] remove not needed include

---
 libraries/plugins/es_objects/es_objects.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp
index 0cd4bc256d..766b7ce886 100644
--- a/libraries/plugins/es_objects/es_objects.cpp
+++ b/libraries/plugins/es_objects/es_objects.cpp
@@ -32,7 +32,6 @@
 #include <graphene/chain/market_object.hpp>
 #include <graphene/chain/asset_object.hpp>
 #include <graphene/chain/account_object.hpp>
-#include <boost/algorithm/string.hpp>
 
 #include <graphene/utilities/elasticsearch.hpp>
 
@@ -74,8 +73,8 @@ class es_objects_plugin_impl
       fc::time_point_sec block_time;
 
    private:
-       template<typename T>
-       void prepareTemplate(T blockchain_object, string index_name);
+      template<typename T>
+      void prepareTemplate(T blockchain_object, string index_name);
 };
 
 bool es_objects_plugin_impl::index_database( const vector<object_id_type>& ids, std::string action)

From 81c276921a8ffef9923b73b47d9f85a4cab69e5c Mon Sep 17 00:00:00 2001
From: Alfredo <oxarbitrage@gmail.com>
Date: Sat, 10 Nov 2018 16:40:39 -0300
Subject: [PATCH 04/38] change variable name

---
 libraries/plugins/es_objects/es_objects.cpp | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp
index 766b7ce886..01ec4722a6 100644
--- a/libraries/plugins/es_objects/es_objects.cpp
+++ b/libraries/plugins/es_objects/es_objects.cpp
@@ -199,16 +199,13 @@ void es_objects_plugin_impl::prepareTemplate(T blockchain_object, string index_n
    }
 
    auto blockchain_object_string = fc::json::to_string(blockchain_object, fc::json::legacy_generator);
-
-   variant j = fc::json::from_string(blockchain_object_string);
-
+   variant blockchain_object_variant = fc::json::from_string(blockchain_object_string);
    fc::mutable_variant_object o;
 
    adaptor_struct adaptor;
-   auto adapted = adaptor.adapt(j.get_object());
+   auto adapted = adaptor.adapt(blockchain_object_variant.get_object());
 
    fc::from_variant(adapted, o, FC_PACK_MAX_DEPTH);
-
    o["object_id"] = string(blockchain_object.id);
    o["block_time"] = block_time;
    o["block_number"] = block_number;

From 5d1c15cdd7faa955463902ea1348ba302eb65681 Mon Sep 17 00:00:00 2001
From: Alfredo <oxarbitrage@gmail.com>
Date: Sun, 25 Nov 2018 21:13:09 -0300
Subject: [PATCH 05/38] add last_vote_time

---
 libraries/chain/account_evaluator.cpp         |  12 +-
 .../include/graphene/chain/account_object.hpp |   3 +
 tests/tests/voting_tests.cpp                  | 125 ++++++++++++++++++
 3 files changed, 137 insertions(+), 3 deletions(-)

diff --git a/libraries/chain/account_evaluator.cpp b/libraries/chain/account_evaluator.cpp
index d550007c76..1ef78dd48c 100644
--- a/libraries/chain/account_evaluator.cpp
+++ b/libraries/chain/account_evaluator.cpp
@@ -57,7 +57,7 @@ void verify_authority_accounts( const database& db, const authority& a )
    }
 }
 
-void verify_account_votes( const database& db, const account_options& options )
+void verify_account_votes( database& db, const account_options& options, const account_id_type account = account_id_type(0))
 {
    // ensure account's votes satisfy requirements
    // NB only the part of vote checking that requires chain state is here,
@@ -117,9 +117,15 @@ void verify_account_votes( const database& db, const account_options& options )
          }
       }
    }
+   if(account != account_id_type(0) &&
+         (options.votes != account(db).options.votes || options.voting_account != account(db).options.voting_account)) {
+      auto &stats_obj = db.get_account_stats_by_owner(account);
+      db.modify(stats_obj, [&](account_statistics_object &obj) {
+         obj.last_vote_time = db.head_block_time();
+      });
+   }
 }
 
-
 void_result account_create_evaluator::do_evaluate( const account_create_operation& op )
 { try {
    database& d = db();
@@ -306,7 +312,7 @@ void_result account_update_evaluator::do_evaluate( const account_update_operatio
    acnt = &o.account(d);
 
    if( o.new_options.valid() )
-      verify_account_votes( d, *o.new_options );
+      verify_account_votes(d, *o.new_options, o.account);
 
    return void_result();
 } FC_CAPTURE_AND_RETHROW( (o) ) }
diff --git a/libraries/chain/include/graphene/chain/account_object.hpp b/libraries/chain/include/graphene/chain/account_object.hpp
index cae9d35984..8e9297a52d 100644
--- a/libraries/chain/include/graphene/chain/account_object.hpp
+++ b/libraries/chain/include/graphene/chain/account_object.hpp
@@ -70,6 +70,8 @@ namespace graphene { namespace chain {
 
          bool is_voting = false; ///< redundately store whether this account is voting for better maintenance performance
 
+         time_point_sec last_vote_time; // add last time voted
+
          /// Whether this account owns some CORE asset and is voting
          inline bool has_some_core_voting() const
          {
@@ -453,6 +455,7 @@ FC_REFLECT_DERIVED( graphene::chain::account_statistics_object,
                     (core_in_balance)
                     (has_cashback_vb)
                     (is_voting)
+                    (last_vote_time)
                     (lifetime_fees_paid)
                     (pending_fees)(pending_vested_fees)
                   )
diff --git a/tests/tests/voting_tests.cpp b/tests/tests/voting_tests.cpp
index 34f39e48c1..b4af210637 100644
--- a/tests/tests/voting_tests.cpp
+++ b/tests/tests/voting_tests.cpp
@@ -414,5 +414,130 @@ BOOST_AUTO_TEST_CASE(invalid_voting_account)
 
    } FC_LOG_AND_RETHROW()
 }
+BOOST_AUTO_TEST_CASE(last_voting_date)
+{
+   try
+   {
+      ACTORS((alice));
+
+      transfer(committee_account, alice_id, asset(100));
+
+      // we are going to vote for this witness
+      auto witness1 = witness_id_type(1)(db);
+
+      auto stats_obj = db.get_account_stats_by_owner(alice_id);
+      BOOST_CHECK_EQUAL(stats_obj.last_vote_time.sec_since_epoch(), 0);
+
+      // alice votes
+      graphene::chain::account_update_operation op;
+      op.account = alice_id;
+      op.new_options = alice.options;
+      op.new_options->votes.insert(witness1.vote_id);
+      trx.operations.push_back(op);
+      sign(trx, alice_private_key);
+      PUSH_TX( db, trx, ~0 );
+
+      auto now = db.head_block_time().sec_since_epoch();
+
+      // last_vote_time is updated for alice
+      stats_obj = db.get_account_stats_by_owner(alice_id);
+      BOOST_CHECK_EQUAL(stats_obj.last_vote_time.sec_since_epoch(), now);
+
+   } FC_LOG_AND_RETHROW()
+}
+BOOST_AUTO_TEST_CASE(last_voting_date_proxy)
+{
+   try
+   {
+      ACTORS((alice)(proxy)(bob));
+
+      transfer(committee_account, alice_id, asset(100));
+      transfer(committee_account, bob_id, asset(200));
+      transfer(committee_account, proxy_id, asset(300));
+
+      generate_block();
+
+      // witness to vote for
+      auto witness1 = witness_id_type(1)(db);
+
+      // alice changes proxy, this is voting activity
+      {
+         graphene::chain::account_update_operation op;
+         op.account = alice_id;
+         op.new_options = alice_id(db).options;
+         op.new_options->voting_account = proxy_id;
+         trx.operations.push_back(op);
+         sign(trx, alice_private_key);
+         PUSH_TX( db, trx, ~0 );
+      }
+      // alice last_vote_time is updated
+      auto alice_stats_obj = db.get_account_stats_by_owner(alice_id);
+      auto now = db.head_block_time().sec_since_epoch();
+      BOOST_CHECK_EQUAL(alice_stats_obj.last_vote_time.sec_since_epoch(), now);
+
+      generate_block();
+
+      // alice update account but no proxy or voting changes are done
+      {
+         graphene::chain::account_update_operation op;
+         op.account = alice_id;
+         op.new_options = alice_id(db).options;
+         trx.operations.push_back(op);
+         sign(trx, alice_private_key);
+         set_expiration( db, trx );
+         PUSH_TX( db, trx, ~0 );
+      }
+      // last_vote_time is not updated
+      now = db.head_block_time().sec_since_epoch();
+      alice_stats_obj = db.get_account_stats_by_owner(alice_id);
+      BOOST_CHECK(alice_stats_obj.last_vote_time.sec_since_epoch() != now);
+
+      generate_block();
+
+      // bob votes
+      {
+         graphene::chain::account_update_operation op;
+         op.account = bob_id;
+         op.new_options = bob_id(db).options;
+         op.new_options->votes.insert(witness1.vote_id);
+         trx.operations.push_back(op);
+         sign(trx, bob_private_key);
+         set_expiration( db, trx );
+         PUSH_TX(db, trx, ~0);
+      }
+
+      // last_vote_time for bob is updated as he voted
+      now = db.head_block_time().sec_since_epoch();
+      auto bob_stats_obj = db.get_account_stats_by_owner(bob_id);
+      BOOST_CHECK_EQUAL(bob_stats_obj.last_vote_time.sec_since_epoch(), now);
+
+      generate_block();
+
+      // proxy votes
+      {
+         graphene::chain::account_update_operation op;
+         op.account = proxy_id;
+         op.new_options = proxy_id(db).options;
+         op.new_options->votes.insert(witness1.vote_id);
+         trx.operations.push_back(op);
+         sign(trx, proxy_private_key);
+         PUSH_TX(db, trx, ~0);
+      }
+
+      // proxy just voted so the last_vote_time is updated
+      now = db.head_block_time().sec_since_epoch();
+      auto proxy_stats_obj = db.get_account_stats_by_owner(proxy_id);
+      BOOST_CHECK_EQUAL(proxy_stats_obj.last_vote_time.sec_since_epoch(), now);
+
+      // alice haves proxy, proxy votes but last_vote_time is not updated for alice
+      alice_stats_obj = db.get_account_stats_by_owner(alice_id);
+      BOOST_CHECK(alice_stats_obj.last_vote_time.sec_since_epoch() != now);
+
+      // bob haves nothing to do with proxy so last_vote_time is not updated
+      bob_stats_obj = db.get_account_stats_by_owner(bob_id);
+      BOOST_CHECK(bob_stats_obj.last_vote_time.sec_since_epoch() != now);
+
+   } FC_LOG_AND_RETHROW()
+}
 
 BOOST_AUTO_TEST_SUITE_END()

From 5e24f652b9e81a8289bdc3c9a99e111e92518ada Mon Sep 17 00:00:00 2001
From: Alfredo <oxarbitrage@gmail.com>
Date: Fri, 30 Nov 2018 19:18:39 -0300
Subject: [PATCH 06/38] change last_vote_time to do_apply

---
 libraries/chain/account_evaluator.cpp | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/libraries/chain/account_evaluator.cpp b/libraries/chain/account_evaluator.cpp
index 1ef78dd48c..23891ef555 100644
--- a/libraries/chain/account_evaluator.cpp
+++ b/libraries/chain/account_evaluator.cpp
@@ -57,7 +57,7 @@ void verify_authority_accounts( const database& db, const authority& a )
    }
 }
 
-void verify_account_votes( database& db, const account_options& options, const account_id_type account = account_id_type(0))
+void verify_account_votes( const database& db, const account_options& options)
 {
    // ensure account's votes satisfy requirements
    // NB only the part of vote checking that requires chain state is here,
@@ -117,13 +117,6 @@ void verify_account_votes( database& db, const account_options& options, const a
          }
       }
    }
-   if(account != account_id_type(0) &&
-         (options.votes != account(db).options.votes || options.voting_account != account(db).options.voting_account)) {
-      auto &stats_obj = db.get_account_stats_by_owner(account);
-      db.modify(stats_obj, [&](account_statistics_object &obj) {
-         obj.last_vote_time = db.head_block_time();
-      });
-   }
 }
 
 void_result account_create_evaluator::do_evaluate( const account_create_operation& op )
@@ -312,7 +305,7 @@ void_result account_update_evaluator::do_evaluate( const account_update_operatio
    acnt = &o.account(d);
 
    if( o.new_options.valid() )
-      verify_account_votes(d, *o.new_options, o.account);
+      verify_account_votes(d, *o.new_options);
 
    return void_result();
 } FC_CAPTURE_AND_RETHROW( (o) ) }
@@ -324,11 +317,16 @@ void_result account_update_evaluator::do_apply( const account_update_operation&
    bool sa_before = acnt->has_special_authority();
 
    // update account statistics
-   if( o.new_options.valid() && o.new_options->is_voting() != acnt->options.is_voting() )
+   if( o.new_options.valid() )
    {
-      d.modify( acnt->statistics( d ), []( account_statistics_object& aso )
+      d.modify( acnt->statistics( d ), [&]( account_statistics_object& aso )
       {
-         aso.is_voting = !aso.is_voting;
+         if(o.new_options->is_voting() != acnt->options.is_voting())
+            aso.is_voting = !aso.is_voting;
+
+         if((o.new_options->votes != acnt->options.votes ||
+               o.new_options->voting_account != acnt->options.voting_account))
+            aso.last_vote_time = d.head_block_time();
       } );
    }
 

From 5688e7b0fa7fc9d4f594e8ec6929c9c0dc8cbb57 Mon Sep 17 00:00:00 2001
From: Alfredo <oxarbitrage@gmail.com>
Date: Fri, 30 Nov 2018 19:22:02 -0300
Subject: [PATCH 07/38] minor spacing changes

---
 libraries/chain/account_evaluator.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/chain/account_evaluator.cpp b/libraries/chain/account_evaluator.cpp
index 23891ef555..086953dcfb 100644
--- a/libraries/chain/account_evaluator.cpp
+++ b/libraries/chain/account_evaluator.cpp
@@ -57,7 +57,7 @@ void verify_authority_accounts( const database& db, const authority& a )
    }
 }
 
-void verify_account_votes( const database& db, const account_options& options)
+void verify_account_votes( const database& db, const account_options& options )
 {
    // ensure account's votes satisfy requirements
    // NB only the part of vote checking that requires chain state is here,
@@ -305,7 +305,7 @@ void_result account_update_evaluator::do_evaluate( const account_update_operatio
    acnt = &o.account(d);
 
    if( o.new_options.valid() )
-      verify_account_votes(d, *o.new_options);
+      verify_account_votes( d, *o.new_options );
 
    return void_result();
 } FC_CAPTURE_AND_RETHROW( (o) ) }

From d8323d4c9d36b7c4651e0e2462ad9fdf451fd812 Mon Sep 17 00:00:00 2001
From: Peter Conrad <cyrano@quisquis.de>
Date: Sun, 2 Dec 2018 10:38:47 +0100
Subject: [PATCH 08/38] Introduced direct_index

---
 libraries/db/include/graphene/db/index.hpp | 108 ++++++++++++++++++++-
 1 file changed, 105 insertions(+), 3 deletions(-)

diff --git a/libraries/db/include/graphene/db/index.hpp b/libraries/db/include/graphene/db/index.hpp
index bcf9b24f13..fd2923f599 100644
--- a/libraries/db/include/graphene/db/index.hpp
+++ b/libraries/db/include/graphene/db/index.hpp
@@ -190,6 +190,95 @@ namespace graphene { namespace db {
          object_database& _db;
    };
 
+   /** @class direct_index
+    *  @brief A secondary index that tracks objects in vectors indexed by object
+    *  id. It is meant for fully (or almost fully) populated indexes only (will
+    *  fail when loading an object_database with large gaps).
+    */
+   template<typename Object, uint8_t chunkbits>
+   class direct_index : public secondary_index
+   {
+      static_assert( chunkbits < 64, "Do you really want arrays with more than 2^63 elements???" );
+
+      private:
+         static const size_t MAX_HOLE = 100;
+         static const size_t _mask = ((1 << chunkbits) - 1);
+         size_t next = 0;
+         vector< vector< const Object* > > content;
+
+      public:
+         direct_index() {
+            FC_ASSERT( (1ULL << chunkbits) > MAX_HOLE, "Small chunkbits is inefficient." );
+         }
+
+         virtual ~direct_index(){}
+
+         virtual void object_inserted( const object& obj )
+         {
+            uint64_t instance = obj.id.instance();
+            if( instance == next )
+            {
+               if( !(next & _mask) )
+               {
+                  content.resize((next >> chunkbits) + 1);
+                  content[next >> chunkbits].reserve( 1 << chunkbits );
+               }
+               next++;
+            }
+            else if( instance < next )
+               FC_ASSERT( !content[instance >> chunkbits][instance & _mask], "Overwriting insert at {id}!", ("id",obj.id) );
+            else // instance > next, allow small "holes"
+            {
+               FC_ASSERT( instance <= next + MAX_HOLE, "Out-of-order insert: {id} > {next}!", ("id",obj.id)("next",next) );
+               if( !(next & _mask) || (next & (~_mask)) != (instance & (~_mask)) )
+               {
+                  content.resize((instance >> chunkbits) + 1);
+                  content[instance >> chunkbits].reserve( 1 << chunkbits );
+               }
+               while( next <= instance )
+               {
+                  content[next >> chunkbits][next & _mask] = nullptr;
+                  next++;
+               }
+            }
+            FC_ASSERT( nullptr != dynamic_cast<const Object*>(&obj), "Wrong object type!" );
+            content[instance >> chunkbits][instance & _mask] = static_cast<const Object*>( &obj );
+         }
+
+         virtual void object_removed( const object& obj )
+         {
+            FC_ASSERT( nullptr != dynamic_cast<const Object*>(&obj), "Wrong object type!" );
+            uint64_t instance = obj.id.instance();
+            FC_ASSERT( instance < next, "Removing out-of-range object: {id} > {next}!", ("id",obj.id)("next",next) );
+            FC_ASSERT( content[instance >> chunkbits][instance & _mask], "Removing non-existent object {id}!", ("id",obj.id) );
+            content[instance >> chunkbits][instance & _mask] = nullptr;
+         }
+
+         template< typename object_id >
+         const Object* find( const object_id& id )const
+         {
+            static_assert( object_id::space_id == Object::space_id, "Space ID mismatch!" );
+            static_assert( object_id::type_id == Object::type_id, "Type_ID mismatch!" );
+            if( id.instance >= next ) return nullptr;
+            return content[id.instance >> chunkbits][id.instance & _mask];
+         };
+
+         template< typename object_id >
+         const Object& get( const object_id& id )const
+         {
+            const Object* ptr = find( id );
+            FC_ASSERT( ptr != nullptr, "Object not found!" );
+            return *ptr;
+         };
+
+         const Object* find( const object_id_type& id )const
+         {
+            FC_ASSERT( id.space() == Object::space_id, "Space ID mismatch!" );
+            FC_ASSERT( id.type() == Object::type_id, "Type_ID mismatch!" );
+            if( id.instance() >= next ) return nullptr;
+            return content[id.instance() >> chunkbits][id.instance() & ((1 << chunkbits) - 1)];
+         };
+   };
 
    /**
     * @class primary_index
@@ -198,14 +287,18 @@ namespace graphene { namespace db {
     *
     *  @see http://en.wikipedia.org/wiki/Curiously_recurring_template_pattern
     */
-   template<typename DerivedIndex>
+   template<typename DerivedIndex, uint8_t DirectBits = 0>
    class primary_index  : public DerivedIndex, public base_primary_index
    {
       public:
          typedef typename DerivedIndex::object_type object_type;
 
          primary_index( object_database& db )
-         :base_primary_index(db),_next_id(object_type::space_id,object_type::type_id,0) {}
+         :base_primary_index(db),_next_id(object_type::space_id,object_type::type_id,0)
+         {
+            if( DirectBits > 0 )
+               _direct_by_id = add_secondary_index< direct_index< object_type, DirectBits > >();
+         }
 
          virtual uint8_t object_space_id()const override
          { return object_type::space_id; }
@@ -217,6 +310,14 @@ namespace graphene { namespace db {
          virtual void           use_next_id()override                    { ++_next_id.number;  }
          virtual void           set_next_id( object_id_type id )override { _next_id = id;      }
 
+         /** @return the object with id or nullptr if not found */
+         virtual const object*  find( object_id_type id )const override
+         {
+            if( DirectBits > 0 )
+               return _direct_by_id->find( id );
+            return DerivedIndex::find( id );
+         }
+
          fc::sha256 get_object_version()const
          {
             std::string desc = "1.0";//get_type_description<object_type>();
@@ -329,7 +430,8 @@ namespace graphene { namespace db {
          }
 
       private:
-         object_id_type _next_id;
+         object_id_type                                 _next_id;
+         const direct_index< object_type, DirectBits >* _direct_by_id = nullptr;
    };
 
 } } // graphene::db

From 6d79167a570e9e73d6adc3c4524296e96b8ec3a8 Mon Sep 17 00:00:00 2001
From: Peter Conrad <cyrano@quisquis.de>
Date: Sun, 2 Dec 2018 10:45:43 +0100
Subject: [PATCH 09/38] Use new derived_index for some

---
 libraries/chain/db_init.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/libraries/chain/db_init.cpp b/libraries/chain/db_init.cpp
index 34d16f4b82..c87d9ec6a6 100644
--- a/libraries/chain/db_init.cpp
+++ b/libraries/chain/db_init.cpp
@@ -182,15 +182,15 @@ void database::initialize_indexes()
    _undo_db.set_max_size( GRAPHENE_MIN_UNDO_HISTORY );
 
    //Protocol object indexes
-   add_index< primary_index<asset_index> >();
+   add_index< primary_index<asset_index, 13> >(); // 8192 assets per chunk
    add_index< primary_index<force_settlement_index> >();
 
-   auto acnt_index = add_index< primary_index<account_index> >();
+   auto acnt_index = add_index< primary_index<account_index, 20> >(); // ~1 million accounts per chunk
    acnt_index->add_secondary_index<account_member_index>();
    acnt_index->add_secondary_index<account_referrer_index>();
 
-   add_index< primary_index<committee_member_index> >();
-   add_index< primary_index<witness_index> >();
+   add_index< primary_index<committee_member_index, 8> >(); // 256 members per chunk
+   add_index< primary_index<witness_index, 10> >(); // 1024 witnesses per chunk
    add_index< primary_index<limit_order_index > >();
    add_index< primary_index<call_order_index > >();
 
@@ -206,10 +206,10 @@ void database::initialize_indexes()
    //Implementation object indexes
    add_index< primary_index<transaction_index                             > >();
    add_index< primary_index<account_balance_index                         > >();
-   add_index< primary_index<asset_bitasset_data_index                     > >();
+   add_index< primary_index<asset_bitasset_data_index,                 13 > >(); // 8192
    add_index< primary_index<simple_index<global_property_object          >> >();
    add_index< primary_index<simple_index<dynamic_global_property_object  >> >();
-   add_index< primary_index<account_stats_index                           > >();
+   add_index< primary_index<account_stats_index,                       20 > >(); // 1 Mi
    add_index< primary_index<simple_index<asset_dynamic_data_object       >> >();
    add_index< primary_index<simple_index<block_summary_object            >> >();
    add_index< primary_index<simple_index<chain_property_object          > > >();

From c367219cbad3c264a9d0c92abeaeb967dd6e133d Mon Sep 17 00:00:00 2001
From: Peter Conrad <cyrano@quisquis.de>
Date: Sun, 2 Dec 2018 10:46:12 +0100
Subject: [PATCH 10/38] Fixed casts

---
 libraries/app/database_api.cpp | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/libraries/app/database_api.cpp b/libraries/app/database_api.cpp
index 6e05f5fd0d..9316cfa4b6 100644
--- a/libraries/app/database_api.cpp
+++ b/libraries/app/database_api.cpp
@@ -547,6 +547,10 @@ vector<vector<account_id_type>> database_api::get_key_references( vector<public_
  */
 vector<vector<account_id_type>> database_api_impl::get_key_references( vector<public_key_type> keys )const
 {
+   const auto& idx = _db.get_index_type<account_index>();
+   const auto& aidx = dynamic_cast<const base_primary_index&>(idx);
+   const auto& refs = aidx.get_secondary_index<graphene::chain::account_member_index>();
+
    vector< vector<account_id_type> > final_result;
    final_result.reserve(keys.size());
 
@@ -566,10 +570,6 @@ vector<vector<account_id_type>> database_api_impl::get_key_references( vector<pu
       subscribe_to_item( a4 );
       subscribe_to_item( a5 );
 
-      const auto& idx = _db.get_index_type<account_index>();
-      const auto& aidx = dynamic_cast<const primary_index<account_index>&>(idx);
-      const auto& refs = aidx.get_secondary_index<graphene::chain::account_member_index>();
-      auto itr = refs.account_to_key_memberships.find(key);
       vector<account_id_type> result;
 
       for( auto& a : {a1,a2,a3,a4,a5} )
@@ -585,6 +585,7 @@ vector<vector<account_id_type>> database_api_impl::get_key_references( vector<pu
           }
       }
 
+      auto itr = refs.account_to_key_memberships.find(key);
       if( itr != refs.account_to_key_memberships.end() )
       {
          result.reserve( itr->second.size() );
@@ -620,7 +621,7 @@ bool database_api_impl::is_public_key_registered(string public_key) const
         return false;
     }
     const auto& idx = _db.get_index_type<account_index>();
-    const auto& aidx = dynamic_cast<const primary_index<account_index>&>(idx);
+    const auto& aidx = dynamic_cast<const base_primary_index&>(idx);
     const auto& refs = aidx.get_secondary_index<graphene::chain::account_member_index>();
     auto itr = refs.account_to_key_memberships.find(key);
     bool is_known = itr != refs.account_to_key_memberships.end();
@@ -755,6 +756,10 @@ std::map<string,full_account> database_api::get_full_accounts( const vector<stri
 
 std::map<std::string, full_account> database_api_impl::get_full_accounts( const vector<std::string>& names_or_ids, bool subscribe)
 {
+   const auto& proposal_idx = _db.get_index_type<proposal_index>();
+   const auto& pidx = dynamic_cast<const base_primary_index&>(proposal_idx);
+   const auto& proposals_by_account = pidx.get_secondary_index<graphene::chain::required_approval_index>();
+
    std::map<std::string, full_account> results;
 
    for (const std::string& account_name_or_id : names_or_ids)
@@ -784,9 +789,6 @@ std::map<std::string, full_account> database_api_impl::get_full_accounts( const
          acnt.cashback_balance = account->cashback_balance(_db);
       }
       // Add the account's proposals
-      const auto& proposal_idx = _db.get_index_type<proposal_index>();
-      const auto& pidx = dynamic_cast<const primary_index<proposal_index>&>(proposal_idx);
-      const auto& proposals_by_account = pidx.get_secondary_index<graphene::chain::required_approval_index>();
       auto  required_approvals_itr = proposals_by_account._account_to_proposals.find( account->id );
       if( required_approvals_itr != proposals_by_account._account_to_proposals.end() )
       {
@@ -869,7 +871,7 @@ vector<account_id_type> database_api::get_account_references( const std::string
 vector<account_id_type> database_api_impl::get_account_references( const std::string account_id_or_name )const
 {
    const auto& idx = _db.get_index_type<account_index>();
-   const auto& aidx = dynamic_cast<const primary_index<account_index>&>(idx);
+   const auto& aidx = dynamic_cast<const base_primary_index&>(idx);
    const auto& refs = aidx.get_secondary_index<graphene::chain::account_member_index>();
    const account_id_type account_id = get_account_from_string(account_id_or_name)->id;
    auto itr = refs.account_to_account_memberships.find(account_id);

From 8999eacf934eb85e1ed9923fdcffa42d365a16e5 Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Mon, 8 Oct 2018 17:19:31 +0200
Subject: [PATCH 11/38] Removed unused _unlinked_index

---
 libraries/chain/fork_database.cpp             | 70 +++----------------
 .../include/graphene/chain/fork_database.hpp  |  3 -
 2 files changed, 11 insertions(+), 62 deletions(-)

diff --git a/libraries/chain/fork_database.cpp b/libraries/chain/fork_database.cpp
index b94a2cdc29..a9b4a95fad 100644
--- a/libraries/chain/fork_database.cpp
+++ b/libraries/chain/fork_database.cpp
@@ -52,7 +52,7 @@ void     fork_database::start_block(signed_block b)
 }
 
 /**
- * Pushes the block into the fork database and caches it if it doesn't link
+ * Pushes the block into the fork database
  *
  */
 shared_ptr<fork_item>  fork_database::push_block(const signed_block& b)
@@ -66,7 +66,6 @@ shared_ptr<fork_item>  fork_database::push_block(const signed_block& b)
       wlog( "Pushing block to fork database that failed to link: ${id}, ${num}", ("id",b.id())("num",b.block_num()) );
       wlog( "Head: ${num}, ${id}", ("num",_head->data.block_num())("id",_head->data.id()) );
       throw;
-      _unlinked_index.insert( item );
    }
    return _head;
 }
@@ -94,35 +93,10 @@ void  fork_database::_push_block(const item_ptr& item)
    {
       _head = item;
       uint32_t min_num = _head->num - std::min( _max_size, _head->num );
-//      ilog( "min block in fork DB ${n}, max_size: ${m}", ("n",min_num)("m",_max_size) );
       auto& num_idx = _index.get<block_num>();
       while( num_idx.size() && (*num_idx.begin())->num < min_num )
          num_idx.erase( num_idx.begin() );
-      
-      _unlinked_index.get<block_num>().erase(_head->num - _max_size);
    }
-   //_push_next( item );
-}
-
-/**
- *  Iterate through the unlinked cache and insert anything that
- *  links to the newly inserted item.  This will start a recursive
- *  set of calls performing a depth-first insertion of pending blocks as
- *  _push_next(..) calls _push_block(...) which will in turn call _push_next
- */
-void fork_database::_push_next( const item_ptr& new_item )
-{
-    auto& prev_idx = _unlinked_index.get<by_previous>();
-
-    auto itr = prev_idx.find( new_item->id );
-    while( itr != prev_idx.end() )
-    {
-       auto tmp = *itr;
-       prev_idx.erase( itr );
-       _push_block( tmp );
-
-       itr = prev_idx.find( new_item->id );
-    }
 }
 
 void fork_database::set_max_size( uint32_t s )
@@ -130,29 +104,15 @@ void fork_database::set_max_size( uint32_t s )
    _max_size = s;
    if( !_head ) return;
 
-   { /// index
-      auto& by_num_idx = _index.get<block_num>();
-      auto itr = by_num_idx.begin();
-      while( itr != by_num_idx.end() )
-      {
-         if( (*itr)->num < std::max(int64_t(0),int64_t(_head->num) - _max_size) )
-            by_num_idx.erase(itr);
-         else
-            break;
-         itr = by_num_idx.begin();
-      }
-   }
-   { /// unlinked_index
-      auto& by_num_idx = _unlinked_index.get<block_num>();
-      auto itr = by_num_idx.begin();
-      while( itr != by_num_idx.end() )
-      {
-         if( (*itr)->num < std::max(int64_t(0),int64_t(_head->num) - _max_size) )
-            by_num_idx.erase(itr);
-         else
-            break;
-         itr = by_num_idx.begin();
-      }
+   auto& by_num_idx = _index.get<block_num>();
+   auto itr = by_num_idx.begin();
+   while( itr != by_num_idx.end() )
+   {
+      if( (*itr)->num < std::max(int64_t(0),int64_t(_head->num) - _max_size) )
+         by_num_idx.erase(itr);
+      else
+         break;
+      itr = by_num_idx.begin();
    }
 }
 
@@ -160,11 +120,7 @@ bool fork_database::is_known_block(const block_id_type& id)const
 {
    auto& index = _index.get<block_id>();
    auto itr = index.find(id);
-   if( itr != index.end() )
-      return true;
-   auto& unlinked_index = _unlinked_index.get<block_id>();
-   auto unlinked_itr = unlinked_index.find(id);
-   return unlinked_itr != unlinked_index.end();
+   return itr != index.end();
 }
 
 item_ptr fork_database::fetch_block(const block_id_type& id)const
@@ -173,10 +129,6 @@ item_ptr fork_database::fetch_block(const block_id_type& id)const
    auto itr = index.find(id);
    if( itr != index.end() )
       return *itr;
-   auto& unlinked_index = _unlinked_index.get<block_id>();
-   auto unlinked_itr = unlinked_index.find(id);
-   if( unlinked_itr != unlinked_index.end() )
-      return *unlinked_itr;
    return item_ptr();
 }
 
diff --git a/libraries/chain/include/graphene/chain/fork_database.hpp b/libraries/chain/include/graphene/chain/fork_database.hpp
index be3991ed80..363a21f2ce 100644
--- a/libraries/chain/include/graphene/chain/fork_database.hpp
+++ b/libraries/chain/include/graphene/chain/fork_database.hpp
@@ -93,12 +93,10 @@ namespace graphene { namespace chain {
 
          struct block_id;
          struct block_num;
-         struct by_previous;
          typedef multi_index_container<
             item_ptr,
             indexed_by<
                hashed_unique<tag<block_id>, member<fork_item, block_id_type, &fork_item::id>, std::hash<fc::ripemd160>>,
-               hashed_non_unique<tag<by_previous>, const_mem_fun<fork_item, block_id_type, &fork_item::previous_id>, std::hash<fc::ripemd160>>,
                ordered_non_unique<tag<block_num>, member<fork_item,uint32_t,&fork_item::num>>
             >
          > fork_multi_index_type;
@@ -112,7 +110,6 @@ namespace graphene { namespace chain {
 
          uint32_t                 _max_size = 1024;
 
-         fork_multi_index_type    _unlinked_index;
          fork_multi_index_type    _index;
          shared_ptr<fork_item>    _head;
    };

From 684998432f41ed1dc4fb03df67fc9103a7800bf5 Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Mon, 8 Oct 2018 21:02:18 +0200
Subject: [PATCH 12/38] Fixed fork_db handling in pop_block

---
 libraries/chain/db_block.cpp      | 18 ++++++++++--------
 libraries/chain/fork_database.cpp |  2 +-
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp
index efc5562a89..d6d083a967 100644
--- a/libraries/chain/db_block.cpp
+++ b/libraries/chain/db_block.cpp
@@ -465,15 +465,17 @@ signed_block database::_generate_block(
 void database::pop_block()
 { try {
    _pending_tx_session.reset();
-   auto head_id = head_block_id();
-   optional<signed_block> head_block = fetch_block_by_id( head_id );
-   GRAPHENE_ASSERT( head_block.valid(), pop_empty_chain, "there are no blocks to pop" );
-
-   _fork_db.pop_block();
+   auto fork_db_head = _fork_db.head();
+   FC_ASSERT( fork_db_head, "Trying to pop() from empty fork database!?" );
+   if( fork_db_head->id == head_block_id() )
+      _fork_db.pop_block();
+   else
+   {
+      fork_db_head = _fork_db.fetch_block( head_block_id() );
+      FC_ASSERT( fork_db_head, "Trying to pop() block that's not in fork database!?" );
+   }
    pop_undo();
-
-   _popped_tx.insert( _popped_tx.begin(), head_block->transactions.begin(), head_block->transactions.end() );
-
+   _popped_tx.insert( _popped_tx.begin(), fork_db_head->data.transactions.begin(), fork_db_head->data.transactions.end() );
 } FC_CAPTURE_AND_RETHROW() }
 
 void database::clear_pending()
diff --git a/libraries/chain/fork_database.cpp b/libraries/chain/fork_database.cpp
index a9b4a95fad..71da27629b 100644
--- a/libraries/chain/fork_database.cpp
+++ b/libraries/chain/fork_database.cpp
@@ -41,7 +41,7 @@ void fork_database::pop_block()
    FC_ASSERT( _head, "no block to pop" );
    auto prev = _head->prev.lock();
    FC_ASSERT( prev, "popping block would leave head block null" );
-    _head = prev;
+   _head = prev;
 }
 
 void     fork_database::start_block(signed_block b)

From 73d8b08504f53f71b80b4e4b676d4fa3459f6406 Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Sun, 7 Oct 2018 14:31:50 +0200
Subject: [PATCH 13/38] Added --revalidate-blockchain

---
 libraries/app/application.cpp                       | 10 +++++++---
 libraries/chain/db_management.cpp                   |  2 ++
 libraries/chain/include/graphene/chain/database.hpp |  4 ++++
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp
index b77da4b711..d1dd2d903e 100644
--- a/libraries/app/application.cpp
+++ b/libraries/app/application.cpp
@@ -394,9 +394,12 @@ void application_impl::startup()
       _chain_db->enable_standby_votes_tracking( _options->at("enable-standby-votes-tracking").as<bool>() );
    }
 
-   if( _options->count("replay-blockchain") )
+   if( _options->count("replay-blockchain") || _options->count("revalidate-blockchain") )
       _chain_db->wipe( _data_dir / "blockchain", false );
 
+   if( _options->count("revalidate-blockchain") )
+      _chain_db->_replay_with_validation = true;
+
    try
    {
       _chain_db->open( _data_dir / "blockchain", initial_state, GRAPHENE_CURRENT_DB_VERSION );
@@ -961,9 +964,10 @@ void application::set_program_options(boost::program_options::options_descriptio
           "Path to create a Genesis State at. If a well-formed JSON file exists at the path, it will be parsed and any "
           "missing fields in a Genesis State will be added, and any unknown fields will be removed. If no file or an "
           "invalid file is found, it will be replaced with an example Genesis State.")
-         ("replay-blockchain", "Rebuild object graph by replaying all blocks")
+         ("replay-blockchain", "Rebuild object graph by replaying all blocks without validatino")
+         ("revalidate-blockchain", "Rebuild object graph by replaying all blocks with full validation")
          ("resync-blockchain", "Delete all blocks and re-sync with network from scratch")
-         ("force-validate", "Force validation of all transactions")
+         ("force-validate", "Force validation of all transactions during normal operation")
          ("genesis-timestamp", bpo::value<uint32_t>(),
           "Replace timestamp from genesis.json with current time plus this many seconds (experts only!)")
          ;
diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp
index 6f627adb55..13dca0fdb7 100644
--- a/libraries/chain/db_management.cpp
+++ b/libraries/chain/db_management.cpp
@@ -82,6 +82,8 @@ void database::reindex( fc::path data_dir )
                    skip_tapos_check |
                    skip_witness_schedule_check |
                    skip_authority_check;
+   if( _replay_with_validation )
+      skip = 0;
 
    size_t total_processed_block_size;
    size_t total_block_size = _block_id_to_block.total_block_size();
diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp
index f0fb8e11c6..c73a0756a9 100644
--- a/libraries/chain/include/graphene/chain/database.hpp
+++ b/libraries/chain/include/graphene/chain/database.hpp
@@ -437,6 +437,10 @@ namespace graphene { namespace chain {
          void                  apply_block( const signed_block& next_block, uint32_t skip = skip_nothing );
          processed_transaction apply_transaction( const signed_transaction& trx, uint32_t skip = skip_nothing );
          operation_result      apply_operation( transaction_evaluation_state& eval_state, const operation& op );
+
+         /** Set to true before open() to force full revalidation during replay */
+         bool                  _replay_with_validation = false;
+
       private:
          void                  _apply_block( const signed_block& next_block );
          processed_transaction _apply_transaction( const signed_transaction& trx );

From 34baaed45a4c3535fd5e23b7aaa8bbf3ada3ef0c Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Thu, 11 Oct 2018 17:56:26 +0200
Subject: [PATCH 14/38] Use externally provided skip flags for replay

---
 libraries/app/application.cpp                 | 19 +++++++++++++++----
 libraries/chain/db_management.cpp             | 11 +----------
 .../chain/include/graphene/chain/database.hpp |  3 ---
 3 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp
index d1dd2d903e..0c27120efd 100644
--- a/libraries/app/application.cpp
+++ b/libraries/app/application.cpp
@@ -26,6 +26,7 @@
 #include <graphene/app/application.hpp>
 #include <graphene/app/plugin.hpp>
 
+#include <graphene/chain/db_with.hpp>
 #include <graphene/chain/genesis_state.hpp>
 #include <graphene/chain/protocol/fee_schedule.hpp>
 #include <graphene/chain/protocol/types.hpp>
@@ -397,12 +398,22 @@ void application_impl::startup()
    if( _options->count("replay-blockchain") || _options->count("revalidate-blockchain") )
       _chain_db->wipe( _data_dir / "blockchain", false );
 
-   if( _options->count("revalidate-blockchain") )
-      _chain_db->_replay_with_validation = true;
-
    try
    {
-      _chain_db->open( _data_dir / "blockchain", initial_state, GRAPHENE_CURRENT_DB_VERSION );
+      // these flags are used in open() only, i. e. during replay
+      uint32_t skip = graphene::chain::database::skip_witness_signature |
+                      graphene::chain::database::skip_block_size_check |
+                      graphene::chain::database::skip_merkle_check |
+                      graphene::chain::database::skip_transaction_signatures |
+                      graphene::chain::database::skip_transaction_dupe_check |
+                      graphene::chain::database::skip_tapos_check |
+                      graphene::chain::database::skip_witness_schedule_check |
+                      graphene::chain::database::skip_authority_check;
+      if( _options->count("revalidate-blockchain") ) // see also handle_block()
+         skip = _options->count("force-validate") ? 0 : graphene::chain::database::skip_transaction_signatures;
+      graphene::chain::detail::with_skip_flags( *_chain_db, skip, [this,&initial_state] () {
+         _chain_db->open( _data_dir / "blockchain", initial_state, GRAPHENE_CURRENT_DB_VERSION );
+      });
    }
    catch( const fc::exception& e )
    {
diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp
index 13dca0fdb7..a86978dc4b 100644
--- a/libraries/chain/db_management.cpp
+++ b/libraries/chain/db_management.cpp
@@ -74,16 +74,7 @@ void database::reindex( fc::path data_dir )
    else
       _undo_db.disable();
 
-   uint32_t skip = skip_witness_signature |
-                   skip_block_size_check |
-                   skip_merkle_check |
-                   skip_transaction_signatures |
-                   skip_transaction_dupe_check |
-                   skip_tapos_check |
-                   skip_witness_schedule_check |
-                   skip_authority_check;
-   if( _replay_with_validation )
-      skip = 0;
+   uint32_t skip = node_properties().skip_flags;
 
    size_t total_processed_block_size;
    size_t total_block_size = _block_id_to_block.total_block_size();
diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp
index c73a0756a9..4fdc984ef0 100644
--- a/libraries/chain/include/graphene/chain/database.hpp
+++ b/libraries/chain/include/graphene/chain/database.hpp
@@ -438,9 +438,6 @@ namespace graphene { namespace chain {
          processed_transaction apply_transaction( const signed_transaction& trx, uint32_t skip = skip_nothing );
          operation_result      apply_operation( transaction_evaluation_state& eval_state, const operation& op );
 
-         /** Set to true before open() to force full revalidation during replay */
-         bool                  _replay_with_validation = false;
-
       private:
          void                  _apply_block( const signed_block& next_block );
          processed_transaction _apply_transaction( const signed_transaction& trx );

From bd8ed044a53c69c2b3326c683b8681806f035059 Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Thu, 11 Oct 2018 21:18:34 +0200
Subject: [PATCH 15/38] Removed skip_authority flag because its used
 interchangably with skip_transaction_signatures

---
 libraries/app/application.cpp                 |  3 +--
 libraries/chain/db_block.cpp                  |  2 +-
 libraries/chain/db_init.cpp                   |  2 +-
 .../chain/include/graphene/chain/database.hpp |  2 +-
 tests/tests/authority_tests.cpp               |  8 ++++----
 tests/tests/bitasset_tests.cpp                |  2 --
 tests/tests/block_tests.cpp                   | 19 ++++++++-----------
 tests/tests/fee_tests.cpp                     |  5 -----
 tests/tests/operation_tests.cpp               |  6 +++---
 tests/tests/operation_tests2.cpp              |  6 +-----
 tests/tests/uia_tests.cpp                     |  4 ++--
 11 files changed, 22 insertions(+), 37 deletions(-)

diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp
index 0c27120efd..eead708ba2 100644
--- a/libraries/app/application.cpp
+++ b/libraries/app/application.cpp
@@ -407,8 +407,7 @@ void application_impl::startup()
                       graphene::chain::database::skip_transaction_signatures |
                       graphene::chain::database::skip_transaction_dupe_check |
                       graphene::chain::database::skip_tapos_check |
-                      graphene::chain::database::skip_witness_schedule_check |
-                      graphene::chain::database::skip_authority_check;
+                      graphene::chain::database::skip_witness_schedule_check;
       if( _options->count("revalidate-blockchain") ) // see also handle_block()
          skip = _options->count("force-validate") ? 0 : graphene::chain::database::skip_transaction_signatures;
       graphene::chain::detail::with_skip_flags( *_chain_db, skip, [this,&initial_state] () {
diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp
index d6d083a967..8c2395ef20 100644
--- a/libraries/chain/db_block.cpp
+++ b/libraries/chain/db_block.cpp
@@ -638,7 +638,7 @@ processed_transaction database::_apply_transaction(const signed_transaction& trx
    const chain_parameters& chain_parameters = get_global_properties().parameters;
    eval_state._trx = &trx;
 
-   if( !(skip & (skip_transaction_signatures | skip_authority_check) ) )
+   if( !(skip & skip_transaction_signatures) )
    {
       auto get_active = [&]( account_id_type id ) { return &id(*this).active; };
       auto get_owner  = [&]( account_id_type id ) { return &id(*this).owner;  };
diff --git a/libraries/chain/db_init.cpp b/libraries/chain/db_init.cpp
index 34d16f4b82..4ca2231571 100644
--- a/libraries/chain/db_init.cpp
+++ b/libraries/chain/db_init.cpp
@@ -235,7 +235,7 @@ void database::init_genesis(const genesis_state_type& genesis_state)
    _undo_db.disable();
    struct auth_inhibitor {
       auth_inhibitor(database& db) : db(db), old_flags(db.node_properties().skip_flags)
-      { db.node_properties().skip_flags |= skip_authority_check; }
+      { db.node_properties().skip_flags |= skip_transaction_signatures; }
       ~auth_inhibitor()
       { db.node_properties().skip_flags = old_flags; }
    private:
diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp
index 4fdc984ef0..3b50f1fdb1 100644
--- a/libraries/chain/include/graphene/chain/database.hpp
+++ b/libraries/chain/include/graphene/chain/database.hpp
@@ -69,7 +69,7 @@ namespace graphene { namespace chain {
             skip_fork_db                = 1 << 3,  ///< used while reindexing
             skip_block_size_check       = 1 << 4,  ///< used when applying locally generated transactions
             skip_tapos_check            = 1 << 5,  ///< used while reindexing -- note this skips expiration check as well
-            skip_authority_check        = 1 << 6,  ///< used while reindexing -- disables any checking of authority on transactions
+            // skip_authority_check        = 1 << 6,  ///< removed because effectively identical to skip_transaction_signatures
             skip_merkle_check           = 1 << 7,  ///< used while reindexing
             skip_assert_evaluation      = 1 << 8,  ///< used while reindexing
             skip_undo_history_check     = 1 << 9,  ///< used while reindexing
diff --git a/tests/tests/authority_tests.cpp b/tests/tests/authority_tests.cpp
index 813cff42d4..0e160c2fd6 100644
--- a/tests/tests/authority_tests.cpp
+++ b/tests/tests/authority_tests.cpp
@@ -1169,7 +1169,7 @@ BOOST_FIXTURE_TEST_CASE( get_required_signatures_test, database_fixture )
          op.owner = auth;
          tx.operations.push_back( op );
          set_expiration( db, tx );
-         PUSH_TX( db, tx, database::skip_transaction_signatures | database::skip_authority_check );
+         PUSH_TX( db, tx, database::skip_transaction_signatures );
       } ;
 
       auto get_active = [&](
@@ -1283,7 +1283,7 @@ BOOST_FIXTURE_TEST_CASE( nonminimal_sig_test, database_fixture )
          op.owner = auth;
          tx.operations.push_back( op );
          set_expiration( db, tx );
-         PUSH_TX( db, tx, database::skip_transaction_signatures | database::skip_authority_check );
+         PUSH_TX( db, tx, database::skip_transaction_signatures );
       } ;
 
       auto get_active = [&](
@@ -1370,7 +1370,7 @@ BOOST_FIXTURE_TEST_CASE( parent_owner_test, database_fixture )
          op.owner = owner;
          tx.operations.push_back( op );
          set_expiration( db, tx );
-         PUSH_TX( db, tx, database::skip_transaction_signatures | database::skip_authority_check );
+         PUSH_TX( db, tx, database::skip_transaction_signatures );
       } ;
 
       auto set_auth = [&](
@@ -1462,7 +1462,7 @@ BOOST_FIXTURE_TEST_CASE( missing_owner_auth_test, database_fixture )
          op.owner = owner;
          tx.operations.push_back( op );
          set_expiration( db, tx );
-         PUSH_TX( db, tx, database::skip_transaction_signatures | database::skip_authority_check );
+         PUSH_TX( db, tx, database::skip_transaction_signatures );
       } ;
 
       auto get_active = [&](
diff --git a/tests/tests/bitasset_tests.cpp b/tests/tests/bitasset_tests.cpp
index 9e9a2814ed..4fc6097650 100644
--- a/tests/tests/bitasset_tests.cpp
+++ b/tests/tests/bitasset_tests.cpp
@@ -448,7 +448,6 @@ BOOST_AUTO_TEST_CASE( hf_890_test )
                  | database::skip_transaction_dupe_check
                  | database::skip_block_size_check
                  | database::skip_tapos_check
-                 | database::skip_authority_check
                  | database::skip_merkle_check
                  ;
    generate_blocks(HARDFORK_615_TIME, true, skip); // get around Graphene issue #615 feed expiration bug
@@ -918,7 +917,6 @@ BOOST_AUTO_TEST_CASE( hf_935_test )
                  | database::skip_transaction_dupe_check
                  | database::skip_block_size_check
                  | database::skip_tapos_check
-                 | database::skip_authority_check
                  | database::skip_merkle_check
                  ;
    generate_blocks( HARDFORK_615_TIME, true, skip ); // get around Graphene issue #615 feed expiration bug
diff --git a/tests/tests/block_tests.cpp b/tests/tests/block_tests.cpp
index e71642726b..14e0295ca6 100644
--- a/tests/tests/block_tests.cpp
+++ b/tests/tests/block_tests.cpp
@@ -687,7 +687,7 @@ BOOST_AUTO_TEST_CASE( duplicate_transactions )
       db2.open(dir2.path(), make_genesis, "TEST");
       BOOST_CHECK( db1.get_chain_id() == db2.get_chain_id() );
 
-      auto skip_sigs = database::skip_transaction_signatures | database::skip_authority_check;
+      auto skip_sigs = database::skip_transaction_signatures;
 
       auto init_account_priv_key  = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) );
       public_key_type init_account_pub_key  = init_account_priv_key.get_public_key();
@@ -766,11 +766,11 @@ BOOST_AUTO_TEST_CASE( tapos )
       trx.operations.push_back(t);
       trx.sign( init_account_priv_key, db1.get_chain_id() );
       //relative_expiration is 1, but ref block is 2 blocks old, so this should fail.
-      GRAPHENE_REQUIRE_THROW(PUSH_TX( db1, trx, database::skip_transaction_signatures | database::skip_authority_check ), fc::exception);
+      GRAPHENE_REQUIRE_THROW(PUSH_TX( db1, trx, database::skip_transaction_signatures ), fc::exception);
       set_expiration( db1, trx );
       trx.clear_signatures();
       trx.sign( init_account_priv_key, db1.get_chain_id() );
-      db1.push_transaction(trx, database::skip_transaction_signatures | database::skip_authority_check);
+      PUSH_TX( db1, trx, database::skip_transaction_signatures );
    } catch (fc::exception& e) {
       edump((e.to_detail_string()));
       throw;
@@ -1056,7 +1056,6 @@ BOOST_FIXTURE_TEST_CASE( pop_block_twice, database_fixture )
       uint32_t skip_flags = (
            database::skip_witness_signature
          | database::skip_transaction_signatures
-         | database::skip_authority_check
          );
 
       const asset_object& core = asset_id_type()(db);
@@ -1231,8 +1230,8 @@ BOOST_FIXTURE_TEST_CASE( transaction_invalidated_in_cache, database_fixture )
       };
 
       // tx's created by ACTORS() have bogus authority, so we need to
-      // skip_authority_check in the block where they're included
-      signed_block b1 = generate_block(db, database::skip_authority_check);
+      // skip_transaction_signatures in the block where they're included
+      signed_block b1 = generate_block(db, database::skip_transaction_signatures);
 
       fc::temp_directory data_dir2( graphene::utilities::temp_directory_path() );
 
@@ -1244,7 +1243,7 @@ BOOST_FIXTURE_TEST_CASE( transaction_invalidated_in_cache, database_fixture )
       {
          optional< signed_block > b = db.fetch_block_by_number( db2.head_block_num()+1 );
          db2.push_block(*b, database::skip_witness_signature
-                           |database::skip_authority_check );
+                           |database::skip_transaction_signatures );
       }
       BOOST_CHECK( db2.get( alice_id ).name == "alice" );
       BOOST_CHECK( db2.get( bob_id ).name == "bob" );
@@ -1253,7 +1252,7 @@ BOOST_FIXTURE_TEST_CASE( transaction_invalidated_in_cache, database_fixture )
       transfer( account_id_type(), alice_id, asset( 1000 ) );
       transfer( account_id_type(),   bob_id, asset( 1000 ) );
       // need to skip authority check here as well for same reason as above
-      db2.push_block(generate_block(db, database::skip_authority_check), database::skip_authority_check);
+      db2.push_block(generate_block(db, database::skip_transaction_signatures), database::skip_transaction_signatures);
 
       BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 1000);
       BOOST_CHECK_EQUAL(db.get_balance(  bob_id, asset_id_type()).amount.value, 1000);
@@ -1478,7 +1477,6 @@ BOOST_FIXTURE_TEST_CASE( update_account_keys, database_fixture )
           database::skip_transaction_dupe_check
         | database::skip_witness_signature
         | database::skip_transaction_signatures
-        | database::skip_authority_check
         ;
 
       // Sam is the creator of accounts
@@ -1606,8 +1604,7 @@ BOOST_FIXTURE_TEST_CASE( update_account_keys, database_fixture )
 
                processed_transaction ptx_create = db.push_transaction( trx,
                   database::skip_transaction_dupe_check |
-                  database::skip_transaction_signatures |
-                  database::skip_authority_check
+                  database::skip_transaction_signatures
                );
                account_id_type alice_account_id =
                   ptx_create.operation_results[0]
diff --git a/tests/tests/fee_tests.cpp b/tests/tests/fee_tests.cpp
index 503c9ebc6e..587814815c 100644
--- a/tests/tests/fee_tests.cpp
+++ b/tests/tests/fee_tests.cpp
@@ -759,7 +759,6 @@ BOOST_AUTO_TEST_CASE( fee_refund_test )
                     | database::skip_transaction_dupe_check
                     | database::skip_block_size_check
                     | database::skip_tapos_check
-                    | database::skip_authority_check
                     | database::skip_merkle_check
                     ;
 
@@ -897,7 +896,6 @@ BOOST_AUTO_TEST_CASE( non_core_fee_refund_test )
                     | database::skip_transaction_dupe_check
                     | database::skip_block_size_check
                     | database::skip_tapos_check
-                    | database::skip_authority_check
                     | database::skip_merkle_check
                     ;
 
@@ -1284,7 +1282,6 @@ BOOST_AUTO_TEST_CASE( hf445_fee_refund_cross_test )
                     | database::skip_transaction_dupe_check
                     | database::skip_block_size_check
                     | database::skip_tapos_check
-                    | database::skip_authority_check
                     | database::skip_merkle_check
                     ;
 
@@ -1791,7 +1788,6 @@ BOOST_AUTO_TEST_CASE( bsip26_fee_refund_test )
                     | database::skip_transaction_dupe_check
                     | database::skip_block_size_check
                     | database::skip_tapos_check
-                    | database::skip_authority_check
                     | database::skip_merkle_check
                     ;
 
@@ -2349,7 +2345,6 @@ BOOST_AUTO_TEST_CASE( bsip26_fee_refund_cross_test )
                     | database::skip_transaction_dupe_check
                     | database::skip_block_size_check
                     | database::skip_tapos_check
-                    | database::skip_authority_check
                     | database::skip_merkle_check
                     ;
 
diff --git a/tests/tests/operation_tests.cpp b/tests/tests/operation_tests.cpp
index 5e4168562d..b00745ff06 100644
--- a/tests/tests/operation_tests.cpp
+++ b/tests/tests/operation_tests.cpp
@@ -1940,7 +1940,7 @@ BOOST_AUTO_TEST_CASE( reserve_asset_test )
          transaction tx;
          tx.operations.push_back( op );
          set_expiration( db, tx );
-         db.push_transaction( tx, database::skip_authority_check | database::skip_tapos_check | database::skip_transaction_signatures );
+         PUSH_TX( db, tx, database::skip_tapos_check | database::skip_transaction_signatures );
       } ;
 
       auto _issue_uia = [&]( const account_object& recipient, asset amount )
@@ -1952,7 +1952,7 @@ BOOST_AUTO_TEST_CASE( reserve_asset_test )
          transaction tx;
          tx.operations.push_back( op );
          set_expiration( db, tx );
-         db.push_transaction( tx, database::skip_authority_check | database::skip_tapos_check | database::skip_transaction_signatures );
+         PUSH_TX( db, tx, database::skip_tapos_check | database::skip_transaction_signatures );
       } ;
 
       int64_t init_balance = 10000;
@@ -2043,7 +2043,7 @@ BOOST_AUTO_TEST_CASE( cover_with_collateral_test )
          transaction tx;
          tx.operations.push_back( op );
          set_expiration( db, tx );
-         db.push_transaction( tx, database::skip_authority_check | database::skip_tapos_check | database::skip_transaction_signatures );
+         PUSH_TX( db, tx, database::skip_tapos_check | database::skip_transaction_signatures );
       } ;
 
       // margin call requirement:  1.75x
diff --git a/tests/tests/operation_tests2.cpp b/tests/tests/operation_tests2.cpp
index 5a876d9418..e0d027da1c 100644
--- a/tests/tests/operation_tests2.cpp
+++ b/tests/tests/operation_tests2.cpp
@@ -565,7 +565,6 @@ BOOST_AUTO_TEST_CASE( withdraw_permission_whitelist_asset_test )
                  | database::skip_transaction_dupe_check
                  | database::skip_block_size_check
                  | database::skip_tapos_check
-                 | database::skip_authority_check
                  | database::skip_merkle_check
                  ;
 
@@ -1052,7 +1051,6 @@ BOOST_AUTO_TEST_CASE( witness_create )
                  | database::skip_transaction_dupe_check
                  | database::skip_block_size_check
                  | database::skip_tapos_check
-                 | database::skip_authority_check
                  | database::skip_merkle_check
                  ;
    generate_block(skip);
@@ -1238,7 +1236,6 @@ BOOST_AUTO_TEST_CASE( global_settle_test )
                  | database::skip_transaction_dupe_check
                  | database::skip_block_size_check
                  | database::skip_tapos_check
-                 | database::skip_authority_check
                  | database::skip_merkle_check
                  ;
 
@@ -1616,7 +1613,6 @@ BOOST_AUTO_TEST_CASE( force_settle_test )
                  | database::skip_transaction_dupe_check
                  | database::skip_block_size_check
                  | database::skip_tapos_check
-                 | database::skip_authority_check
                  | database::skip_merkle_check
                  ;
 
@@ -2058,7 +2054,7 @@ BOOST_AUTO_TEST_CASE(zero_second_vbo)
          transaction tx;
          tx.operations.push_back( op );
          set_expiration( db, tx );
-         db.push_transaction( tx, database::skip_authority_check | database::skip_tapos_check | database::skip_transaction_signatures );
+         PUSH_TX( db, tx, database::skip_tapos_check | database::skip_transaction_signatures );
       }
       enable_fees();
       upgrade_to_lifetime_member(alice_id);
diff --git a/tests/tests/uia_tests.cpp b/tests/tests/uia_tests.cpp
index 7b44ef4c04..56b2116a44 100644
--- a/tests/tests/uia_tests.cpp
+++ b/tests/tests/uia_tests.cpp
@@ -377,7 +377,7 @@ BOOST_AUTO_TEST_CASE( transfer_restricted_test )
          transaction tx;
          tx.operations.push_back( op );
          set_expiration( db, tx );
-         PUSH_TX( db, tx, database::skip_authority_check | database::skip_tapos_check | database::skip_transaction_signatures );
+         PUSH_TX( db, tx, database::skip_tapos_check | database::skip_transaction_signatures );
       } ;
 
       const asset_object& uia = create_user_issued_asset( "TXRX", sam, transfer_restricted );
@@ -396,7 +396,7 @@ BOOST_AUTO_TEST_CASE( transfer_restricted_test )
          transaction tx;
          tx.operations.push_back( op );
          set_expiration( db, tx );
-         PUSH_TX( db, tx, database::skip_authority_check | database::skip_tapos_check | database::skip_transaction_signatures );
+         PUSH_TX( db, tx, database::skip_tapos_check | database::skip_transaction_signatures );
       } ;
 
       BOOST_TEST_MESSAGE( "Enable transfer_restricted, send fails" );

From 56dfa9c35abf85f97caf4d71b80a336c69ff6c20 Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Thu, 11 Oct 2018 21:20:47 +0200
Subject: [PATCH 16/38] Removed unused skip_validate flag introduced in
 #dca5c95 2016-01-04

---
 libraries/chain/db_block.cpp                        | 3 +--
 libraries/chain/include/graphene/chain/database.hpp | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp
index 8c2395ef20..9b85dad148 100644
--- a/libraries/chain/db_block.cpp
+++ b/libraries/chain/db_block.cpp
@@ -623,8 +623,7 @@ processed_transaction database::_apply_transaction(const signed_transaction& trx
 { try {
    uint32_t skip = get_node_properties().skip_flags;
 
-   if( true || !(skip&skip_validate) )   /* issue #505 explains why this skip_flag is disabled */
-      trx.validate();
+   trx.validate();
 
    auto& trx_idx = get_mutable_index_type<transaction_index>();
    const chain_id_type& chain_id = get_chain_id();
diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp
index 3b50f1fdb1..c7d35cc13f 100644
--- a/libraries/chain/include/graphene/chain/database.hpp
+++ b/libraries/chain/include/graphene/chain/database.hpp
@@ -73,8 +73,7 @@ namespace graphene { namespace chain {
             skip_merkle_check           = 1 << 7,  ///< used while reindexing
             skip_assert_evaluation      = 1 << 8,  ///< used while reindexing
             skip_undo_history_check     = 1 << 9,  ///< used while reindexing
-            skip_witness_schedule_check = 1 << 10, ///< used while reindexing
-            skip_validate               = 1 << 11  ///< used prior to checkpoint, skips validate() call on transaction
+            skip_witness_schedule_check = 1 << 10 ///< used while reindexing
          };
 
          /**

From 1a78e1b2be2982b5138df6b18965a79c747dd10b Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Thu, 11 Oct 2018 22:07:43 +0200
Subject: [PATCH 17/38] Add warning when revalidating with checkpoints

---
 libraries/app/application.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp
index eead708ba2..057e6368de 100644
--- a/libraries/app/application.cpp
+++ b/libraries/app/application.cpp
@@ -409,7 +409,11 @@ void application_impl::startup()
                       graphene::chain::database::skip_tapos_check |
                       graphene::chain::database::skip_witness_schedule_check;
       if( _options->count("revalidate-blockchain") ) // see also handle_block()
+      {
+         if( !loaded_checkpoints.empty() )
+            wlog( "Warning - revalidate will not validate before last checkpoint" );
          skip = _options->count("force-validate") ? 0 : graphene::chain::database::skip_transaction_signatures;
+      }
       graphene::chain::detail::with_skip_flags( *_chain_db, skip, [this,&initial_state] () {
          _chain_db->open( _data_dir / "blockchain", initial_state, GRAPHENE_CURRENT_DB_VERSION );
       });

From cf2d6e7b29977e29988f01e6dfcd49fd412a22d1 Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Fri, 12 Oct 2018 18:00:43 +0200
Subject: [PATCH 18/38] Make distinction between skip_flag cases clearer

---
 libraries/app/application.cpp                 | 22 ++++++++++++-------
 .../graphene/debug_witness/debug_witness.hpp  |  2 +-
 .../include/graphene/witness/witness.hpp      |  2 +-
 3 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp
index 057e6368de..3858c1871c 100644
--- a/libraries/app/application.cpp
+++ b/libraries/app/application.cpp
@@ -401,19 +401,25 @@ void application_impl::startup()
    try
    {
       // these flags are used in open() only, i. e. during replay
-      uint32_t skip = graphene::chain::database::skip_witness_signature |
-                      graphene::chain::database::skip_block_size_check |
-                      graphene::chain::database::skip_merkle_check |
-                      graphene::chain::database::skip_transaction_signatures |
-                      graphene::chain::database::skip_transaction_dupe_check |
-                      graphene::chain::database::skip_tapos_check |
-                      graphene::chain::database::skip_witness_schedule_check;
+      uint32_t skip;
       if( _options->count("revalidate-blockchain") ) // see also handle_block()
       {
          if( !loaded_checkpoints.empty() )
             wlog( "Warning - revalidate will not validate before last checkpoint" );
-         skip = _options->count("force-validate") ? 0 : graphene::chain::database::skip_transaction_signatures;
+         if( _options->count("force-validate") )
+            skip = graphene::chain::database::skip_nothing;
+         else
+            skip = graphene::chain::database::skip_transaction_signatures;
       }
+      else // no revalidate, skip most checks
+         skip = graphene::chain::database::skip_witness_signature |
+                graphene::chain::database::skip_block_size_check |
+                graphene::chain::database::skip_merkle_check |
+                graphene::chain::database::skip_transaction_signatures |
+                graphene::chain::database::skip_transaction_dupe_check |
+                graphene::chain::database::skip_tapos_check |
+                graphene::chain::database::skip_witness_schedule_check;
+
       graphene::chain::detail::with_skip_flags( *_chain_db, skip, [this,&initial_state] () {
          _chain_db->open( _data_dir / "blockchain", initial_state, GRAPHENE_CURRENT_DB_VERSION );
       });
diff --git a/libraries/plugins/debug_witness/include/graphene/debug_witness/debug_witness.hpp b/libraries/plugins/debug_witness/include/graphene/debug_witness/debug_witness.hpp
index 907d26ae9a..22c71236b5 100644
--- a/libraries/plugins/debug_witness/include/graphene/debug_witness/debug_witness.hpp
+++ b/libraries/plugins/debug_witness/include/graphene/debug_witness/debug_witness.hpp
@@ -58,7 +58,7 @@ class debug_witness_plugin : public graphene::app::plugin {
 
    boost::program_options::variables_map _options;
 
-   std::map<chain::public_key_type, fc::ecc::private_key> _private_keys;
+   std::map<chain::public_key_type, fc::ecc::private_key, chain::pubkey_comparator> _private_keys;
 
    std::shared_ptr< std::ofstream > _json_object_stream;
    boost::signals2::scoped_connection _applied_block_conn;
diff --git a/libraries/plugins/witness/include/graphene/witness/witness.hpp b/libraries/plugins/witness/include/graphene/witness/witness.hpp
index 9292b55e9f..0d2eab27ac 100644
--- a/libraries/plugins/witness/include/graphene/witness/witness.hpp
+++ b/libraries/plugins/witness/include/graphene/witness/witness.hpp
@@ -81,7 +81,7 @@ class witness_plugin : public graphene::app::plugin {
    uint32_t _required_witness_participation = 33 * GRAPHENE_1_PERCENT;
    uint32_t _production_skip_flags = graphene::chain::database::skip_nothing;
 
-   std::map<chain::public_key_type, fc::ecc::private_key> _private_keys;
+   std::map<chain::public_key_type, fc::ecc::private_key, chain::pubkey_comparator> _private_keys;
    std::set<chain::witness_id_type> _witnesses;
    fc::future<void> _block_production_task;
 

From d644f200af36b33a3aa95acc4211acb2acc4861b Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Fri, 26 Oct 2018 15:56:13 +0200
Subject: [PATCH 19/38] Prove that irrelevant signature detection depends on
 sorting order

---
 tests/tests/authority_tests.cpp | 38 +++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/tests/tests/authority_tests.cpp b/tests/tests/authority_tests.cpp
index 0e160c2fd6..310a992060 100644
--- a/tests/tests/authority_tests.cpp
+++ b/tests/tests/authority_tests.cpp
@@ -1677,4 +1677,42 @@ BOOST_AUTO_TEST_CASE( issue_214 )
    BOOST_CHECK_EQUAL( top.amount.amount.value, get_balance( bob_id, top.amount.asset_id ) );
 } FC_LOG_AND_RETHROW() }
 
+BOOST_AUTO_TEST_CASE( irrelevant_signatures )
+{ try {
+   ACTORS( (alice)(bob) );
+   fund( alice );
+
+   // PK: BTS4vsFgTXJcGQMKCFayF2hrNRfYcKjNZ6Mzk8aw9M4zuWfscPhzE, A: BTSGfxPKKLj6tdTUB7i3mHsd2m7QvPLPy2YA
+   const fc::ecc::private_key test2 = fc::ecc::private_key::regenerate( fc::sha256::hash( std::string( "test-2" ) ) );
+   const public_key_type test2_pub( test2.get_public_key() );
+
+   // PK: BTS7FXC7S9UH7HEH8QiuJ8Xv1NRJJZd1GomALLm9ffjtH95Tb2ZQB, A: BTSBajRqmdrXqmDpZhJ8sgkGagdeXneHFVeM
+   const fc::ecc::private_key test3 = fc::ecc::private_key::regenerate( fc::sha256::hash( std::string( "test-3" ) ) );
+   const public_key_type test3_pub( test3.get_public_key() );
+
+   BOOST_REQUIRE( test2_pub.key_data < test3_pub.key_data );
+   BOOST_REQUIRE( address( test3_pub ) < address( test2_pub ) );
+
+   account_update_operation auo;
+   auo.account = alice_id;
+   auo.active = authority( 2, test2_pub, 2, test3_pub, 1 );
+
+   trx.clear();
+   set_expiration( db, trx );
+   trx.operations.push_back( auo );
+   sign( trx, alice_private_key );
+   PUSH_TX( db, trx );
+   trx.clear();
+
+   transfer_operation to;
+   to.amount = asset( 1 );
+   to.from = alice_id;
+   to.to = bob_id;
+   trx.operations.push_back( to );
+   sign( trx, test2 );
+   sign( trx, test3 );
+   PUSH_TX( db, trx );
+
+} FC_LOG_AND_RETHROW() }
+
 BOOST_AUTO_TEST_SUITE_END()

From 0c4c13364bb6a29fd06a254c9f9e9d1071412f07 Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Thu, 25 Oct 2018 17:43:49 +0200
Subject: [PATCH 20/38] Fixed typo

---
 libraries/app/application.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp
index 3858c1871c..882f8f7a8f 100644
--- a/libraries/app/application.cpp
+++ b/libraries/app/application.cpp
@@ -984,7 +984,7 @@ void application::set_program_options(boost::program_options::options_descriptio
           "Path to create a Genesis State at. If a well-formed JSON file exists at the path, it will be parsed and any "
           "missing fields in a Genesis State will be added, and any unknown fields will be removed. If no file or an "
           "invalid file is found, it will be replaced with an example Genesis State.")
-         ("replay-blockchain", "Rebuild object graph by replaying all blocks without validatino")
+         ("replay-blockchain", "Rebuild object graph by replaying all blocks without validation")
          ("revalidate-blockchain", "Rebuild object graph by replaying all blocks with full validation")
          ("resync-blockchain", "Delete all blocks and re-sync with network from scratch")
          ("force-validate", "Force validation of all transactions during normal operation")

From e5ba271c264c2c4173e12f37db9d0def1e068ab8 Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Thu, 27 Sep 2018 20:24:30 +0200
Subject: [PATCH 21/38] Parallelize loading/saving object_database

---
 libraries/db/object_database.cpp | 17 +++++++++++++++--
 libraries/fc                     |  2 +-
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/libraries/db/object_database.cpp b/libraries/db/object_database.cpp
index fdde0fed6b..5b026c08cf 100644
--- a/libraries/db/object_database.cpp
+++ b/libraries/db/object_database.cpp
@@ -25,6 +25,7 @@
 
 #include <fc/io/raw.hpp>
 #include <fc/container/flat.hpp>
+#include <fc/thread/parallel.hpp>
 #include <fc/uint128.hpp>
 
 namespace graphene { namespace db {
@@ -72,14 +73,20 @@ void object_database::flush()
 {
 //   ilog("Save object_database in ${d}", ("d", _data_dir));
    fc::create_directories( _data_dir / "object_database.tmp" / "lock" );
+   std::vector<fc::future<void>> tasks;
+   tasks.reserve(200);
    for( uint32_t space = 0; space < _index.size(); ++space )
    {
       fc::create_directories( _data_dir / "object_database.tmp" / fc::to_string(space) );
       const auto types = _index[space].size();
       for( uint32_t type = 0; type  <  types; ++type )
          if( _index[space][type] )
-            _index[space][type]->save( _data_dir / "object_database.tmp" / fc::to_string(space)/fc::to_string(type) );
+            tasks.push_back( fc::do_parallel( [this,space,type] () {
+               _index[space][type]->save( _data_dir / "object_database.tmp" / fc::to_string(space)/fc::to_string(type) );
+            } ) );
    }
+   for( auto& task : tasks )
+      task.wait();
    fc::remove_all( _data_dir / "object_database.tmp" / "lock" );
    if( fc::exists( _data_dir / "object_database" ) )
       fc::rename( _data_dir / "object_database", _data_dir / "object_database.old" );
@@ -103,11 +110,17 @@ void object_database::open(const fc::path& data_dir)
        wlog("Ignoring locked object_database");
        return;
    }
+   std::vector<fc::future<void>> tasks;
+   tasks.reserve(200);
    ilog("Opening object database from ${d} ...", ("d", data_dir));
    for( uint32_t space = 0; space < _index.size(); ++space )
       for( uint32_t type = 0; type  < _index[space].size(); ++type )
          if( _index[space][type] )
-            _index[space][type]->open( _data_dir / "object_database" / fc::to_string(space)/fc::to_string(type) );
+            tasks.push_back( fc::do_parallel( [this,space,type] () {
+               _index[space][type]->open( _data_dir / "object_database" / fc::to_string(space)/fc::to_string(type) );
+            } ) );
+   for( auto& task : tasks )
+      task.wait();
    ilog( "Done opening object database." );
 
 } FC_CAPTURE_AND_RETHROW( (data_dir) ) }
diff --git a/libraries/fc b/libraries/fc
index 9cce60c917..0468884ea6 160000
--- a/libraries/fc
+++ b/libraries/fc
@@ -1 +1 @@
-Subproject commit 9cce60c91773ad99cfdfa42c5e86ba6ceb3f3ee9
+Subproject commit 0468884ea675afe3a16ffc61371672fecf6e7dde

From 033ddea8cdab350cbfed6f87de86dfb59dd4d959 Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Thu, 4 Oct 2018 16:43:26 +0200
Subject: [PATCH 22/38] Introduce precomputable_transaction, and
 clearable_block in tests

---
 .../graphene/chain/protocol/transaction.hpp   | 47 ++++++++++++++-----
 libraries/chain/protocol/transaction.cpp      | 46 ++++++++++++------
 tests/common/database_fixture.cpp             |  7 +++
 tests/common/database_fixture.hpp             |  5 ++
 tests/tests/authority_tests.cpp               |  1 -
 tests/tests/basic_tests.cpp                   | 11 ++++-
 tests/tests/block_tests.cpp                   |  4 +-
 7 files changed, 91 insertions(+), 30 deletions(-)

diff --git a/libraries/chain/include/graphene/chain/protocol/transaction.hpp b/libraries/chain/include/graphene/chain/protocol/transaction.hpp
index 95c3996135..2f67d5dd60 100644
--- a/libraries/chain/include/graphene/chain/protocol/transaction.hpp
+++ b/libraries/chain/include/graphene/chain/protocol/transaction.hpp
@@ -85,9 +85,9 @@ namespace graphene { namespace chain {
       extensions_type    extensions;
 
       /// Calculate the digest for a transaction
-      digest_type         digest()const;
-      transaction_id_type id()const;
-      void                validate() const;
+      digest_type                        digest()const;
+      virtual const transaction_id_type& id()const;
+      virtual void                       validate() const;
       /// Calculate the digest used for signature validation
       digest_type         sig_digest( const chain_id_type& chain_id )const;
 
@@ -113,6 +113,10 @@ namespace graphene { namespace chain {
       }
 
       void get_required_authorities( flat_set<account_id_type>& active, flat_set<account_id_type>& owner, vector<authority>& other )const;
+
+   protected:
+      digest_type sig_digest( const chain_id_type& chain_id )const;
+      mutable transaction_id_type _tx_id_buffer;
    };
 
    /**
@@ -176,7 +180,7 @@ namespace graphene { namespace chain {
        *       otherwise, the @ref chain_id parameter will be ignored, and
        *       @ref signees will be returned directly.
        */
-      const flat_set<public_key_type>& get_signature_keys( const chain_id_type& chain_id )const;
+      virtual const flat_set<public_key_type>& get_signature_keys( const chain_id_type& chain_id )const;
 
       /** Signatures */
       vector<signature_type> signatures;
@@ -184,11 +188,31 @@ namespace graphene { namespace chain {
       /** Public keys extracted from signatures */
       mutable flat_set<public_key_type> signees;
 
-      /// Removes all operations, signatures and signees
-      void clear() { operations.clear(); signatures.clear(); signees.clear(); }
+      /// Removes all operations and signatures
+      void clear() { operations.clear(); signatures.clear(); }
+
+      /// Removes all signatures
+      void clear_signatures() { signatures.clear(); }
+   protected:
+      /** Public keys extracted from signatures */
+      mutable flat_set<public_key_type> _signees;
+   };
+
+   /** This represents a signed transaction that will never have its operations,
+    *  signatures etc. modified again, after initial creation. It is therefore
+    *  safe to cache results from various calls.
+    */
+   class precomputable_transaction : public signed_transaction {
+   public:
+      precomputable_transaction() {}
+      precomputable_transaction( const signed_transaction& tx ) : signed_transaction(tx) {};
+      precomputable_transaction( signed_transaction&& tx ) : signed_transaction( std::move(tx) ) {};
 
-      /// Removes all signatures and signees
-      void clear_signatures() { signatures.clear(); signees.clear(); }
+      virtual const transaction_id_type&       id()const;
+      virtual void                             validate() const;
+      virtual const flat_set<public_key_type>& get_signature_keys( const chain_id_type& chain_id )const;
+   protected:
+      mutable bool _validated = false;
    };
 
    void verify_authority( const vector<operation>& ops, const flat_set<public_key_type>& sigs,
@@ -212,10 +236,10 @@ namespace graphene { namespace chain {
     *  If an operation did not create any new object IDs then 0
     *  should be returned.
     */
-   struct processed_transaction : public signed_transaction
+   struct processed_transaction : public precomputable_transaction
    {
       processed_transaction( const signed_transaction& trx = signed_transaction() )
-         : signed_transaction(trx){}
+         : precomputable_transaction(trx){}
 
       vector<operation_result> operation_results;
 
@@ -229,4 +253,5 @@ namespace graphene { namespace chain {
 FC_REFLECT( graphene::chain::transaction, (ref_block_num)(ref_block_prefix)(expiration)(operations)(extensions) )
 // Note: not reflecting signees field for backward compatibility; in addition, it should not be in p2p messages
 FC_REFLECT_DERIVED( graphene::chain::signed_transaction, (graphene::chain::transaction), (signatures) )
-FC_REFLECT_DERIVED( graphene::chain::processed_transaction, (graphene::chain::signed_transaction), (operation_results) )
+FC_REFLECT_DERIVED( graphene::chain::precomputable_transaction, (graphene::chain::signed_transaction), )
+FC_REFLECT_DERIVED( graphene::chain::processed_transaction, (graphene::chain::precomputable_transaction), (operation_results) )
diff --git a/libraries/chain/protocol/transaction.cpp b/libraries/chain/protocol/transaction.cpp
index 4bb7ce8378..04e3cb4914 100644
--- a/libraries/chain/protocol/transaction.cpp
+++ b/libraries/chain/protocol/transaction.cpp
@@ -60,19 +60,17 @@ void transaction::validate() const
       operation_validate(op);
 }
 
-graphene::chain::transaction_id_type graphene::chain::transaction::id() const
+const graphene::chain::transaction_id_type& graphene::chain::transaction::id() const
 {
    auto h = digest();
-   transaction_id_type result;
-   memcpy(result._hash, h._hash, std::min(sizeof(result), sizeof(h)));
-   return result;
+   memcpy(_tx_id_buffer._hash, h._hash, std::min(sizeof(_tx_id_buffer), sizeof(h)));
+   return _tx_id_buffer;
 }
 
 const signature_type& graphene::chain::signed_transaction::sign(const private_key_type& key, const chain_id_type& chain_id)
 {
    digest_type h = sig_digest( chain_id );
    signatures.push_back(key.sign_compact(h));
-   signees.clear(); // Clear signees since it may be inconsistent after added a new signature
    return signatures.back();
 }
 
@@ -307,20 +305,17 @@ const flat_set<public_key_type>& signed_transaction::get_signature_keys( const c
 { try {
    // Strictly we should check whether the given chain ID is same as the one used to initialize the `signees` field.
    // However, we don't pass in another chain ID so far, for better performance, we skip the check.
-   if( signees.empty() && !signatures.empty() )
+   auto d = sig_digest( chain_id );
+   flat_set<public_key_type> result;
+   for( const auto&  sig : signatures )
    {
-      auto d = sig_digest( chain_id );
-      flat_set<public_key_type> result;
-      for( const auto&  sig : signatures )
-      {
-         GRAPHENE_ASSERT(
-            result.insert( fc::ecc::public_key(sig,d) ).second,
+      GRAPHENE_ASSERT(
+         result.insert( fc::ecc::public_key(sig,d) ).second,
             tx_duplicate_sig,
             "Duplicate Signature detected" );
-      }
-      signees = std::move( result );
    }
-   return signees;
+   _signees = std::move( result );
+   return _signees;
 } FC_CAPTURE_AND_RETHROW() }
 
 
@@ -386,6 +381,27 @@ set<public_key_type> signed_transaction::minimize_required_signatures(
    return set<public_key_type>( result.begin(), result.end() );
 }
 
+const transaction_id_type& precomputable_transaction::id()const
+{
+   if( !_tx_id_buffer._hash[0] )
+      transaction::id();
+   return _tx_id_buffer;
+}
+
+void precomputable_transaction::validate() const
+{
+   if( _validated ) return;
+   transaction::validate();
+   _validated = true;
+}
+
+const flat_set<public_key_type>& precomputable_transaction::get_signature_keys( const chain_id_type& chain_id )const
+{
+   if( _signees.empty() )
+      signed_transaction::get_signature_keys( chain_id );
+   return _signees;
+}
+
 void signed_transaction::verify_authority(
    const chain_id_type& chain_id,
    const std::function<const authority*(account_id_type)>& get_active,
diff --git a/tests/common/database_fixture.cpp b/tests/common/database_fixture.cpp
index 534c03de33..2e05f71017 100644
--- a/tests/common/database_fixture.cpp
+++ b/tests/common/database_fixture.cpp
@@ -55,6 +55,13 @@ namespace graphene { namespace chain {
 using std::cout;
 using std::cerr;
 
+void clearable_block::clear()
+{
+   _calculated_merkle_root = checksum_type();
+   _signee = fc::ecc::public_key();
+   _block_id = block_id_type();
+}
+
 database_fixture::database_fixture()
    : app(), db( *app.chain_database() )
 {
diff --git a/tests/common/database_fixture.hpp b/tests/common/database_fixture.hpp
index 78f7b65d19..72744ff7e5 100644
--- a/tests/common/database_fixture.hpp
+++ b/tests/common/database_fixture.hpp
@@ -170,6 +170,11 @@ extern uint32_t GRAPHENE_TESTING_GENESIS_TIMESTAMP;
 
 namespace graphene { namespace chain {
 
+class clearable_block : public signed_block {
+public:
+   void clear();
+};
+
 struct database_fixture {
    // the reason we use an app is to exercise the indexes of built-in
    //   plugins
diff --git a/tests/tests/authority_tests.cpp b/tests/tests/authority_tests.cpp
index 310a992060..573d609edf 100644
--- a/tests/tests/authority_tests.cpp
+++ b/tests/tests/authority_tests.cpp
@@ -1057,7 +1057,6 @@ BOOST_FIXTURE_TEST_CASE( bogus_signature, database_fixture )
       PUSH_TX( db,  trx, skip  );
 
       trx.operations.push_back( xfer_op );
-      trx.signees.clear(); // signees should be invalidated
       BOOST_TEST_MESSAGE( "Invalidating Alices Signature" );
       // Alice's signature is now invalid
       GRAPHENE_REQUIRE_THROW( PUSH_TX( db,  trx, skip  ), fc::exception );
diff --git a/tests/tests/basic_tests.cpp b/tests/tests/basic_tests.cpp
index 2157dde75a..97d03ed41b 100644
--- a/tests/tests/basic_tests.cpp
+++ b/tests/tests/basic_tests.cpp
@@ -408,7 +408,7 @@ BOOST_AUTO_TEST_CASE( scaled_precision )
 
 BOOST_AUTO_TEST_CASE( merkle_root )
 {
-   signed_block block;
+   clearable_block block;
    vector<processed_transaction> tx;
    vector<digest_type> t;
    const uint32_t num_tx = 10;
@@ -444,6 +444,7 @@ BOOST_AUTO_TEST_CASE( merkle_root )
    dA = d(t[0], t[1]);
 
    block.transactions.push_back( tx[1] );
+   block.clear();
    BOOST_CHECK( block.calculate_merkle_root() == c(dA) );
 
    /*
@@ -458,6 +459,7 @@ BOOST_AUTO_TEST_CASE( merkle_root )
    dI = d(dA, dB);
 
    block.transactions.push_back( tx[2] );
+   block.clear();
    BOOST_CHECK( block.calculate_merkle_root() == c(dI) );
 
    /*
@@ -472,6 +474,7 @@ BOOST_AUTO_TEST_CASE( merkle_root )
    dI = d(dA, dB);
 
    block.transactions.push_back( tx[3] );
+   block.clear();
    BOOST_CHECK( block.calculate_merkle_root() == c(dI) );
 
    /*
@@ -489,6 +492,7 @@ BOOST_AUTO_TEST_CASE( merkle_root )
    dM = d(dI, dJ);
 
    block.transactions.push_back( tx[4] );
+   block.clear();
    BOOST_CHECK( block.calculate_merkle_root() == c(dM) );
 
    /*
@@ -506,6 +510,7 @@ BOOST_AUTO_TEST_CASE( merkle_root )
    dM = d(dI, dJ);
 
    block.transactions.push_back( tx[5] );
+   block.clear();
    BOOST_CHECK( block.calculate_merkle_root() == c(dM) );
 
    /*
@@ -523,6 +528,7 @@ BOOST_AUTO_TEST_CASE( merkle_root )
    dM = d(dI, dJ);
 
    block.transactions.push_back( tx[6] );
+   block.clear();
    BOOST_CHECK( block.calculate_merkle_root() == c(dM) );
 
    /*
@@ -540,6 +546,7 @@ BOOST_AUTO_TEST_CASE( merkle_root )
    dM = d(dI, dJ);
 
    block.transactions.push_back( tx[7] );
+   block.clear();
    BOOST_CHECK( block.calculate_merkle_root() == c(dM) );
 
    /*
@@ -560,6 +567,7 @@ BOOST_AUTO_TEST_CASE( merkle_root )
    dO = d(dM, dN);
 
    block.transactions.push_back( tx[8] );
+   block.clear();
    BOOST_CHECK( block.calculate_merkle_root() == c(dO) );
 
    /*
@@ -580,6 +588,7 @@ BOOST_AUTO_TEST_CASE( merkle_root )
    dO = d(dM, dN);
 
    block.transactions.push_back( tx[9] );
+   block.clear();
    BOOST_CHECK( block.calculate_merkle_root() == c(dO) );
 }
 
diff --git a/tests/tests/block_tests.cpp b/tests/tests/block_tests.cpp
index 14e0295ca6..1890dc38b8 100644
--- a/tests/tests/block_tests.cpp
+++ b/tests/tests/block_tests.cpp
@@ -80,11 +80,12 @@ BOOST_AUTO_TEST_CASE( block_database_test )
       FC_ASSERT( !bdb.is_open() );
       bdb.open( data_dir.path() );
 
-      signed_block b;
+      clearable_block b;
       for( uint32_t i = 0; i < 5; ++i )
       {
          if( i > 0 ) b.previous = b.id();
          b.witness = witness_id_type(i+1);
+         b.clear();
          bdb.store( b.id(), b );
 
          auto fetch = bdb.fetch_by_number( b.block_num() );
@@ -976,7 +977,6 @@ BOOST_FIXTURE_TEST_CASE( double_sign_check, database_fixture )
 
    BOOST_TEST_MESSAGE( "Verify that signing once with the proper key passes" );
    trx.signatures.pop_back();
-   trx.signees.clear(); // signees should be invalidated
    db.push_transaction(trx, 0);
 
 } FC_LOG_AND_RETHROW() }

From d2d8b29c097071b15ee311bf675512d589651b1b Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Thu, 27 Sep 2018 22:09:44 +0200
Subject: [PATCH 23/38] Preprocess blocks + transactions in parallel, working
 version

---
 libraries/app/api.cpp                         |  13 ++-
 libraries/app/application.cpp                 |  19 ++--
 libraries/app/application_impl.hxx            |   4 +
 libraries/app/include/graphene/app/api.hpp    |   6 +-
 libraries/chain/db_block.cpp                  |  71 ++++++++++++
 libraries/chain/db_management.cpp             | 107 +++++++++++-------
 .../chain/include/graphene/chain/database.hpp |  24 ++++
 .../include/graphene/chain/protocol/base.hpp  |   3 +
 .../include/graphene/chain/protocol/block.hpp |  20 +++-
 .../graphene/chain/protocol/transaction.hpp   |  14 +--
 libraries/chain/protocol/block.cpp            |  64 ++++++-----
 libraries/chain/protocol/operations.cpp       |   6 +
 libraries/chain/protocol/transaction.cpp      |   2 +-
 .../include/graphene/net/core_messages.hpp    |   4 +-
 .../delayed_node/delayed_node_plugin.cpp      |   1 +
 15 files changed, 253 insertions(+), 105 deletions(-)

diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp
index dbe6043e50..f3cffa70ff 100644
--- a/libraries/app/api.cpp
+++ b/libraries/app/api.cpp
@@ -159,18 +159,18 @@ namespace graphene { namespace app {
        }
     }
 
-    void network_broadcast_api::broadcast_transaction(const signed_transaction& trx)
+    void network_broadcast_api::broadcast_transaction(const precomputable_transaction& trx)
     {
-       trx.validate();
+       _app.chain_database()->precompute_parallel( trx ).wait();
        _app.chain_database()->push_transaction(trx);
        if( _app.p2p_node() != nullptr )
           _app.p2p_node()->broadcast_transaction(trx);
     }
 
-    fc::variant network_broadcast_api::broadcast_transaction_synchronous(const signed_transaction& trx)
+    fc::variant network_broadcast_api::broadcast_transaction_synchronous(const precomputable_transaction& trx)
     {
        fc::promise<fc::variant>::ptr prom( new fc::promise<fc::variant>() );
-       broadcast_transaction_with_callback( [=]( const fc::variant& v ){
+       broadcast_transaction_with_callback( [prom]( const fc::variant& v ){
         prom->set_value(v);
        }, trx );
 
@@ -179,14 +179,15 @@ namespace graphene { namespace app {
 
     void network_broadcast_api::broadcast_block( const signed_block& b )
     {
+       _app.chain_database()->precompute_parallel( b ).wait();
        _app.chain_database()->push_block(b);
        if( _app.p2p_node() != nullptr )
           _app.p2p_node()->broadcast( net::block_message( b ));
     }
 
-    void network_broadcast_api::broadcast_transaction_with_callback(confirmation_callback cb, const signed_transaction& trx)
+    void network_broadcast_api::broadcast_transaction_with_callback(confirmation_callback cb, const precomputable_transaction& trx)
     {
-       trx.validate();
+       _app.chain_database()->precompute_parallel( trx ).wait();
        _callbacks[trx.id()] = cb;
        _app.chain_database()->push_transaction(trx);
        if( _app.p2p_node() != nullptr )
diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp
index 882f8f7a8f..cc7828034f 100644
--- a/libraries/app/application.cpp
+++ b/libraries/app/application.cpp
@@ -540,13 +540,17 @@ bool application_impl::handle_block(const graphene::net::block_message& blk_msg,
    FC_ASSERT( (latency.count()/1000) > -5000, "Rejecting block with timestamp in the future" );
 
    try {
-      // TODO: in the case where this block is valid but on a fork that's too old for us to switch to,
-      // you can help the network code out by throwing a block_older_than_undo_history exception.
-      // when the net code sees that, it will stop trying to push blocks from that chain, but
-      // leave that peer connected so that they can get sync blocks from us
-      bool result = _chain_db->push_block( blk_msg.block,
-                                           (_is_block_producer | _force_validate) ?
-                                              database::skip_nothing : database::skip_transaction_signatures );
+      const uint32_t skip = (_is_block_producer | _force_validate) ?
+                               database::skip_nothing : database::skip_transaction_signatures;
+      bool result = valve.do_serial( [this,&blk_msg,skip] () {
+         _chain_db->precompute_parallel( blk_msg.block, skip ).wait();
+      }, [this,&blk_msg,skip] () {
+         // TODO: in the case where this block is valid but on a fork that's too old for us to switch to,
+         // you can help the network code out by throwing a block_older_than_undo_history exception.
+         // when the net code sees that, it will stop trying to push blocks from that chain, but
+         // leave that peer connected so that they can get sync blocks from us
+         return _chain_db->push_block( blk_msg.block, skip );
+      });
 
       // the block was accepted, so we now know all of the transactions contained in the block
       if (!sync_mode)
@@ -596,6 +600,7 @@ void application_impl::handle_transaction(const graphene::net::trx_message& tran
       trx_count = 0;
    }
 
+   _chain_db->precompute_parallel( transaction_message.trx ).wait();
    _chain_db->push_transaction( transaction_message.trx );
 } FC_CAPTURE_AND_RETHROW( (transaction_message) ) }
 
diff --git a/libraries/app/application_impl.hxx b/libraries/app/application_impl.hxx
index 5b0c543728..9f601bce79 100644
--- a/libraries/app/application_impl.hxx
+++ b/libraries/app/application_impl.hxx
@@ -1,6 +1,8 @@
 #pragma once
 
 #include <fc/network/http/websocket.hpp>
+#include <fc/thread/parallel.hpp>
+
 #include <graphene/app/application.hpp>
 #include <graphene/app/api_access.hpp>
 #include <graphene/chain/genesis_state.hpp>
@@ -194,6 +196,8 @@ class application_impl : public net::node_delegate
       std::map<string, std::shared_ptr<abstract_plugin>> _available_plugins;
 
       bool _is_finished_syncing = false;
+   private:
+      fc::serial_valve valve;
    };
 
 }}} // namespace graphene namespace app namespace detail
diff --git a/libraries/app/include/graphene/app/api.hpp b/libraries/app/include/graphene/app/api.hpp
index aea35be08b..af34bbaf73 100644
--- a/libraries/app/include/graphene/app/api.hpp
+++ b/libraries/app/include/graphene/app/api.hpp
@@ -266,19 +266,19 @@ namespace graphene { namespace app {
           * The transaction will be checked for validity in the local database prior to broadcasting. If it fails to
           * apply locally, an error will be thrown and the transaction will not be broadcast.
           */
-         void broadcast_transaction(const signed_transaction& trx);
+         void broadcast_transaction(const precomputable_transaction& trx);
 
          /** this version of broadcast transaction registers a callback method that will be called when the transaction is
           * included into a block.  The callback method includes the transaction id, block number, and transaction number in the
           * block.
           */
-         void broadcast_transaction_with_callback( confirmation_callback cb, const signed_transaction& trx);
+         void broadcast_transaction_with_callback( confirmation_callback cb, const precomputable_transaction& trx);
 
          /** this version of broadcast transaction registers a callback method that will be called when the transaction is
           * included into a block.  The callback method includes the transaction id, block number, and transaction number in the
           * block.
           */
-         fc::variant broadcast_transaction_synchronous(const signed_transaction& trx);
+         fc::variant broadcast_transaction_synchronous(const precomputable_transaction& trx);
 
          /**
           * @brief Broadcast a signed block to the network
diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp
index 9b85dad148..296d88e649 100644
--- a/libraries/chain/db_block.cpp
+++ b/libraries/chain/db_block.cpp
@@ -37,6 +37,7 @@
 #include <graphene/chain/exceptions.hpp>
 #include <graphene/chain/evaluator.hpp>
 
+#include <fc/thread/parallel.hpp>
 #include <fc/smart_ref_impl.hpp>
 
 namespace graphene { namespace chain {
@@ -751,4 +752,74 @@ bool database::before_last_checkpoint()const
    return (_checkpoints.size() > 0) && (_checkpoints.rbegin()->first >= head_block_num());
 }
 
+
+static const uint32_t skip_expensive = database::skip_transaction_signatures | database::skip_witness_signature
+                                       | database::skip_merkle_check | database::skip_transaction_dupe_check;
+
+template<typename Trx>
+void database::_precompute_parallel( const Trx* trx, const size_t count, const uint32_t skip )const
+{
+   for( size_t i = 0; i < count; ++i, ++trx )
+   {
+      trx->validate(); // TODO - parallelize wrt confidential operations
+      if( !(skip&skip_transaction_dupe_check) )
+         trx->id();
+      if( !(skip&skip_transaction_signatures) )
+         trx->get_signature_keys( get_chain_id() );
+   }
+}
+
+void database::_precompute_parallel( const signed_block& block, const uint32_t skip )const
+{ try {
+   const bool cheap = (skip & skip_expensive) == skip_expensive;
+   std::vector<fc::future<void>> workers;
+   if( !block.transactions.empty() )
+   {
+      if( cheap )
+         _precompute_parallel( &block.transactions[0], block.transactions.size(), skip );
+      else
+      {
+         uint32_t chunks = fc::asio::default_io_service_scope::get_num_threads();
+         if( !chunks ) return;
+         uint32_t chunk_size = block.transactions.size() / chunks;
+         if( chunks * chunk_size < block.transactions.size() )
+            chunk_size++;
+         workers.reserve( chunks + 1 );
+         for( size_t base = 0; base < block.transactions.size(); base += chunk_size )
+            workers.push_back( fc::do_parallel( [this,&block,base,chunk_size,skip] () {
+               _precompute_parallel( &block.transactions[base],
+                                     base + chunk_size < block.transactions.size() ? chunk_size : block.transactions.size() - base,
+                                     skip );
+            }) );
+      }
+   }
+
+   if( !(skip&skip_witness_signature) )
+      workers.push_back( fc::do_parallel( [&block] () { block.signee(); } ) );
+   if( !(skip&skip_merkle_check) )
+      block.calculate_merkle_root();
+   block.id();
+   for( auto& worker : workers )
+      worker.wait();
+} FC_LOG_AND_RETHROW() }
+
+fc::future<void> database::precompute_parallel( const signed_block& block, const uint32_t skip )const
+{
+   if( block.transactions.empty() || (skip & skip_expensive) == skip_expensive )
+   {
+      _precompute_parallel( block, skip );
+      return fc::future< void >( fc::promise< void >::ptr( new fc::promise< void >( true ) ) );
+   }
+   return fc::do_parallel([this,&block,skip] () {
+      _precompute_parallel( block, skip );
+   });
+}
+
+fc::future<void> database::precompute_parallel( const precomputable_transaction& trx )const
+{
+   return fc::do_parallel([this,&trx] () {
+      _precompute_parallel( &trx, 1, skip_nothing );
+   });
+}
+
 } }
diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp
index a86978dc4b..9167768f48 100644
--- a/libraries/chain/db_management.cpp
+++ b/libraries/chain/db_management.cpp
@@ -35,6 +35,8 @@
 #include <fstream>
 #include <functional>
 #include <iostream>
+#include <queue>
+#include <tuple>
 
 namespace graphene { namespace chain {
 
@@ -76,59 +78,76 @@ void database::reindex( fc::path data_dir )
 
    uint32_t skip = node_properties().skip_flags;
 
-   size_t total_processed_block_size;
    size_t total_block_size = _block_id_to_block.total_block_size();
    const auto& gpo = get_global_properties();
-   for( uint32_t i = head_block_num() + 1; i <= last_block_num; ++i )
+   std::queue< std::tuple< size_t, signed_block, fc::future< void > > > blocks;
+   uint32_t next_block_num = head_block_num() + 1;
+   uint32_t i = next_block_num;
+   while( next_block_num <= last_block_num || !blocks.empty() )
    {
-      if( i % 10000 == 0 ) 
+      if( next_block_num <= last_block_num && blocks.size() < 20 )
       {
-         total_processed_block_size = _block_id_to_block.blocks_current_position();
-
-         ilog(
-            "   [by size: ${size}%   ${processed} of ${total}]   [by num: ${num}%   ${i} of ${last}]",
-            ("size", double(total_processed_block_size) / total_block_size * 100)
-            ("processed", total_processed_block_size)
-            ("total", total_block_size)
-            ("num", double(i*100)/last_block_num)
-            ("i", i)
-            ("last", last_block_num)
-         );
-      }
-      if( i == flush_point )
-      {
-         ilog( "Writing database to disk at block ${i}", ("i",i) );
-         flush();
-         ilog( "Done" );
-      }
-      if( head_block_time() >= last_block->timestamp - gpo.parameters.maximum_time_until_expiration )
-         skip &= ~skip_transaction_dupe_check;
-      fc::optional< signed_block > block = _block_id_to_block.fetch_by_number(i);
-      if( !block.valid() )
-      {
-         wlog( "Reindexing terminated due to gap:  Block ${i} does not exist!", ("i", i) );
-         uint32_t dropped_count = 0;
-         while( true )
+         const size_t processed_block_size = _block_id_to_block.blocks_current_position();
+         fc::optional< signed_block > block = _block_id_to_block.fetch_by_number( next_block_num++ );
+         if( block.valid() )
          {
-            fc::optional< block_id_type > last_id = _block_id_to_block.last_id();
-            // this can trigger if we attempt to e.g. read a file that has block #2 but no block #1
-            if( !last_id.valid() )
-               break;
-            // we've caught up to the gap
-            if( block_header::num_from_id( *last_id ) <= i )
-               break;
-            _block_id_to_block.remove( *last_id );
-            dropped_count++;
+            if( block->timestamp >= last_block->timestamp - gpo.parameters.maximum_time_until_expiration )
+               skip &= ~skip_transaction_dupe_check;
+            blocks.emplace( processed_block_size, std::move(*block), fc::future<void>() );
+            std::get<2>(blocks.back()) = precompute_parallel( std::get<1>(blocks.back()), skip );
+         }
+         else
+         {
+            wlog( "Reindexing terminated due to gap:  Block ${i} does not exist!", ("i", i) );
+            uint32_t dropped_count = 0;
+            while( true )
+            {
+               fc::optional< block_id_type > last_id = _block_id_to_block.last_id();
+               // this can trigger if we attempt to e.g. read a file that has block #2 but no block #1
+               if( !last_id.valid() )
+                  break;
+               // we've caught up to the gap
+               if( block_header::num_from_id( *last_id ) <= i )
+                  break;
+               _block_id_to_block.remove( *last_id );
+               dropped_count++;
+            }
+            wlog( "Dropped ${n} blocks from after the gap", ("n", dropped_count) );
+            next_block_num = last_block_num + 1; // don't load more blocks
          }
-         wlog( "Dropped ${n} blocks from after the gap", ("n", dropped_count) );
-         break;
       }
-      if( i < undo_point )
-         apply_block( *block, skip );
       else
       {
-         _undo_db.enable();
-         push_block( *block, skip );
+         std::get<2>(blocks.front()).wait();
+         const signed_block& block = std::get<1>(blocks.front());
+
+         if( i % 10000 == 0 )
+         {
+            ilog(
+               "   [by size: ${size}%   ${processed} of ${total}]   [by num: ${num}%   ${i} of ${last}]",
+               ("size", double(std::get<0>(blocks.front())) / total_block_size * 100)
+               ("processed", std::get<0>(blocks.front()))
+               ("total", total_block_size)
+               ("num", double(i*100)/last_block_num)
+               ("i", i)
+               ("last", last_block_num)
+            );
+         }
+         if( i == flush_point )
+         {
+            ilog( "Writing database to disk at block ${i}", ("i",i) );
+            flush();
+            ilog( "Done" );
+         }
+         if( i < undo_point )
+            apply_block( block, skip );
+         else
+         {
+            _undo_db.enable();
+            push_block( block, skip );
+         }
+         blocks.pop();
+         i++;
       }
    }
    _undo_db.enable();
diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp
index c7d35cc13f..db2f38e5de 100644
--- a/libraries/chain/include/graphene/chain/database.hpp
+++ b/libraries/chain/include/graphene/chain/database.hpp
@@ -415,6 +415,30 @@ namespace graphene { namespace chain {
          /// Enable or disable tracking of votes of standby witnesses and committee members
          inline void enable_standby_votes_tracking(bool enable)  { _track_standby_votes = enable; }
 
+         /** Precomputes digests, signatures and operation validations depending
+          *  on skip flags. "Expensive" computations may be done in a parallel
+          *  thread.
+          *
+          * @param block the block to preprocess
+          * @param skip indicates which computations can be skipped
+          * @return a future that will resolve to the input block with
+          *         precomputations applied
+          */
+         fc::future<void> precompute_parallel( const signed_block& block, const uint32_t skip = skip_nothing )const;
+
+         /** Precomputes digests, signatures and operation validations.
+          *  "Expensive" computations may be done in a parallel thread.
+          *
+          * @param trx the transaction to preprocess
+          * @return a future that will resolve to the input transaction with
+          *         precomputations applied
+          */
+         fc::future<void> precompute_parallel( const precomputable_transaction& trx )const;
+   private:
+         template<typename Trx>
+         void _precompute_parallel( const Trx* trx, const size_t count, const uint32_t skip )const;
+         void _precompute_parallel( const signed_block& block, const uint32_t skip )const;
+
    protected:
          //Mark pop_undo() as protected -- we do not want outside calling pop_undo(); it should call pop_block() instead
          void pop_undo() { object_database::pop_undo(); }
diff --git a/libraries/chain/include/graphene/chain/protocol/base.hpp b/libraries/chain/include/graphene/chain/protocol/base.hpp
index 52240b934a..73209a1861 100644
--- a/libraries/chain/include/graphene/chain/protocol/base.hpp
+++ b/libraries/chain/include/graphene/chain/protocol/base.hpp
@@ -27,6 +27,8 @@
 #include <graphene/chain/protocol/asset.hpp>
 #include <graphene/chain/protocol/authority.hpp>
 
+#include <fc/thread/future.hpp>
+
 namespace graphene { namespace chain {
 
    /**
@@ -94,6 +96,7 @@ namespace graphene { namespace chain {
       void get_required_active_authorities( flat_set<account_id_type>& )const{}
       void get_required_owner_authorities( flat_set<account_id_type>& )const{}
       void validate()const{}
+      fc::optional< fc::future<void> > validate_parallel( uint32_t skip )const;
 
       static uint64_t calculate_data_fee( uint64_t bytes, uint64_t price_per_kbyte );
    };
diff --git a/libraries/chain/include/graphene/chain/protocol/block.hpp b/libraries/chain/include/graphene/chain/protocol/block.hpp
index 98e627c928..aa8c46052f 100644
--- a/libraries/chain/include/graphene/chain/protocol/block.hpp
+++ b/libraries/chain/include/graphene/chain/protocol/block.hpp
@@ -26,8 +26,9 @@
 
 namespace graphene { namespace chain {
 
-   struct block_header
+   class block_header
    {
+   public:
       digest_type                   digest()const;
       block_id_type                 previous;
       uint32_t                      block_num()const { return num_from_id(previous) + 1; }
@@ -41,20 +42,27 @@ namespace graphene { namespace chain {
       static uint32_t num_from_id(const block_id_type& id);
    };
 
-   struct signed_block_header : public block_header
+   class signed_block_header : public block_header
    {
-      block_id_type              id()const;
-      fc::ecc::public_key        signee()const;
+   public:
+      const block_id_type&       id()const;
+      const fc::ecc::public_key& signee()const;
       void                       sign( const fc::ecc::private_key& signer );
       bool                       validate_signee( const fc::ecc::public_key& expected_signee )const;
 
       signature_type             witness_signature;
+   protected:
+      mutable fc::ecc::public_key _signee;
+      mutable block_id_type       _block_id;
    };
 
-   struct signed_block : public signed_block_header
+   class signed_block : public signed_block_header
    {
-      checksum_type calculate_merkle_root()const;
+   public:
+      const checksum_type& calculate_merkle_root()const;
       vector<processed_transaction> transactions;
+   protected:
+      mutable checksum_type   _calculated_merkle_root;
    };
 
 } } // graphene::chain
diff --git a/libraries/chain/include/graphene/chain/protocol/transaction.hpp b/libraries/chain/include/graphene/chain/protocol/transaction.hpp
index 2f67d5dd60..02f00799c6 100644
--- a/libraries/chain/include/graphene/chain/protocol/transaction.hpp
+++ b/libraries/chain/include/graphene/chain/protocol/transaction.hpp
@@ -62,8 +62,9 @@ namespace graphene { namespace chain {
    /**
     *  @brief groups operations that should be applied atomically
     */
-   struct transaction
+   class transaction
    {
+   public:
       /**
        * Least significant 16 bits from the reference block number. If @ref relative_expiration is zero, this field
        * must be zero as well.
@@ -89,7 +90,6 @@ namespace graphene { namespace chain {
       virtual const transaction_id_type& id()const;
       virtual void                       validate() const;
       /// Calculate the digest used for signature validation
-      digest_type         sig_digest( const chain_id_type& chain_id )const;
 
       void set_expiration( fc::time_point_sec expiration_time );
       void set_reference_block( const block_id_type& reference_block );
@@ -122,8 +122,9 @@ namespace graphene { namespace chain {
    /**
     *  @brief adds a signature to a transaction
     */
-   struct signed_transaction : public transaction
+   class signed_transaction : public transaction
    {
+   public:
       signed_transaction( const transaction& trx = transaction() )
          : transaction(trx){}
 
@@ -185,13 +186,10 @@ namespace graphene { namespace chain {
       /** Signatures */
       vector<signature_type> signatures;
 
-      /** Public keys extracted from signatures */
-      mutable flat_set<public_key_type> signees;
-
-      /// Removes all operations and signatures
+      /** Removes all operations and signatures */
       void clear() { operations.clear(); signatures.clear(); }
 
-      /// Removes all signatures
+      /** Removes all signatures */
       void clear_signatures() { signatures.clear(); }
    protected:
       /** Public keys extracted from signatures */
diff --git a/libraries/chain/protocol/block.cpp b/libraries/chain/protocol/block.cpp
index d32365dd08..9fdf4707eb 100644
--- a/libraries/chain/protocol/block.cpp
+++ b/libraries/chain/protocol/block.cpp
@@ -37,19 +37,23 @@ namespace graphene { namespace chain {
       return fc::endian_reverse_u32(id._hash[0]);
    }
 
-   block_id_type signed_block_header::id()const
+   const block_id_type& signed_block_header::id()const
    {
-      auto tmp = fc::sha224::hash( *this );
-      tmp._hash[0] = fc::endian_reverse_u32(block_num()); // store the block num in the ID, 160 bits is plenty for the hash
-      static_assert( sizeof(tmp._hash[0]) == 4, "should be 4 bytes" );
-      block_id_type result;
-      memcpy(result._hash, tmp._hash, std::min(sizeof(result), sizeof(tmp)));
-      return result;
+      if( !_block_id._hash[0] )
+      {
+         auto tmp = fc::sha224::hash( *this );
+         tmp._hash[0] = fc::endian_reverse_u32(block_num()); // store the block num in the ID, 160 bits is plenty for the hash
+         static_assert( sizeof(tmp._hash[0]) == 4, "should be 4 bytes" );
+         memcpy(_block_id._hash, tmp._hash, std::min(sizeof(_block_id), sizeof(tmp)));
+      }
+      return _block_id;
    }
 
-   fc::ecc::public_key signed_block_header::signee()const
+   const fc::ecc::public_key& signed_block_header::signee()const
    {
-      return fc::ecc::public_key( witness_signature, digest(), true/*enforce canonical*/ );
+      if( !_signee.valid() )
+         _signee = fc::ecc::public_key( witness_signature, digest(), true/*enforce canonical*/ );
+      return _signee;
    }
 
    void signed_block_header::sign( const fc::ecc::private_key& signer )
@@ -62,31 +66,35 @@ namespace graphene { namespace chain {
       return signee() == expected_signee;
    }
 
-   checksum_type signed_block::calculate_merkle_root()const
+   const checksum_type& signed_block::calculate_merkle_root()const
    {
+      static const checksum_type empty_checksum;
       if( transactions.size() == 0 ) 
-         return checksum_type();
-
-      vector<digest_type> ids;
-      ids.resize( transactions.size() );
-      for( uint32_t i = 0; i < transactions.size(); ++i )
-         ids[i] = transactions[i].merkle_digest();
+         return empty_checksum;
 
-      vector<digest_type>::size_type current_number_of_hashes = ids.size();
-      while( current_number_of_hashes > 1 )
+      if( !_calculated_merkle_root._hash[0] )
       {
-         // hash ID's in pairs
-         uint32_t i_max = current_number_of_hashes - (current_number_of_hashes&1);
-         uint32_t k = 0;
+         vector<digest_type> ids;
+         ids.resize( transactions.size() );
+         for( uint32_t i = 0; i < transactions.size(); ++i )
+            ids[i] = transactions[i].merkle_digest();
 
-         for( uint32_t i = 0; i < i_max; i += 2 )
-            ids[k++] = digest_type::hash( std::make_pair( ids[i], ids[i+1] ) );
+         vector<digest_type>::size_type current_number_of_hashes = ids.size();
+         while( current_number_of_hashes > 1 )
+         {
+            // hash ID's in pairs
+            uint32_t i_max = current_number_of_hashes - (current_number_of_hashes&1);
+            uint32_t k = 0;
 
-         if( current_number_of_hashes&1 )
-            ids[k++] = ids[i_max];
-         current_number_of_hashes = k;
+            for( uint32_t i = 0; i < i_max; i += 2 )
+               ids[k++] = digest_type::hash( std::make_pair( ids[i], ids[i+1] ) );
+
+            if( current_number_of_hashes&1 )
+               ids[k++] = ids[i_max];
+            current_number_of_hashes = k;
+         }
+         _calculated_merkle_root = checksum_type::hash( ids[0] );
       }
-      return checksum_type::hash( ids[0] );
+      return _calculated_merkle_root;
    }
-
 } }
diff --git a/libraries/chain/protocol/operations.cpp b/libraries/chain/protocol/operations.cpp
index 40a37eba3a..48a65f6fed 100644
--- a/libraries/chain/protocol/operations.cpp
+++ b/libraries/chain/protocol/operations.cpp
@@ -32,6 +32,12 @@ uint64_t base_operation::calculate_data_fee( uint64_t bytes, uint64_t price_per_
    return result.to_uint64();
 }
 
+fc::optional< fc::future<void> > base_operation::validate_parallel( uint32_t skip )const
+{
+   validate();
+   return fc::optional< fc::future<void> >();
+}
+
 void balance_claim_operation::validate()const
 {
    FC_ASSERT( fee == asset() );
diff --git a/libraries/chain/protocol/transaction.cpp b/libraries/chain/protocol/transaction.cpp
index 04e3cb4914..c1dd817693 100644
--- a/libraries/chain/protocol/transaction.cpp
+++ b/libraries/chain/protocol/transaction.cpp
@@ -60,7 +60,7 @@ void transaction::validate() const
       operation_validate(op);
 }
 
-const graphene::chain::transaction_id_type& graphene::chain::transaction::id() const
+const transaction_id_type& transaction::id() const
 {
    auto h = digest();
    memcpy(_tx_id_buffer._hash, h._hash, std::min(sizeof(_tx_id_buffer), sizeof(h)));
diff --git a/libraries/net/include/graphene/net/core_messages.hpp b/libraries/net/include/graphene/net/core_messages.hpp
index 8af0c3443c..76f74bd253 100644
--- a/libraries/net/include/graphene/net/core_messages.hpp
+++ b/libraries/net/include/graphene/net/core_messages.hpp
@@ -95,9 +95,9 @@ namespace graphene { namespace net {
    {
       static const core_message_type_enum type;
 
-      signed_transaction trx;
+      graphene::chain::precomputable_transaction trx;
       trx_message() {}
-      trx_message(signed_transaction transaction) :
+      trx_message(graphene::chain::signed_transaction transaction) :
         trx(std::move(transaction))
       {}
    };
diff --git a/libraries/plugins/delayed_node/delayed_node_plugin.cpp b/libraries/plugins/delayed_node/delayed_node_plugin.cpp
index 24a46cc066..dd1d4705e6 100644
--- a/libraries/plugins/delayed_node/delayed_node_plugin.cpp
+++ b/libraries/plugins/delayed_node/delayed_node_plugin.cpp
@@ -105,6 +105,7 @@ void delayed_node_plugin::sync_with_trusted_node()
          fc::optional<graphene::chain::signed_block> block = my->database_api->get_block( db.head_block_num()+1 );
          FC_ASSERT(block, "Trusted node claims it has blocks it doesn't actually have.");
          ilog("Pushing block #${n}", ("n", block->block_num()));
+         db.precompute_parallel( *block, graphene::chain::database::skip_nothing ).wait();
          db.push_block(*block);
          synced_blocks++;
       }

From 9946bdefd093499684deb77ba01992e261c661da Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Fri, 5 Oct 2018 17:53:40 +0200
Subject: [PATCH 24/38] Get rid of possibly uninitialized local variable

---
 libraries/chain/db_block.cpp | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp
index 296d88e649..4873c65ed5 100644
--- a/libraries/chain/db_block.cpp
+++ b/libraries/chain/db_block.cpp
@@ -628,12 +628,8 @@ processed_transaction database::_apply_transaction(const signed_transaction& trx
 
    auto& trx_idx = get_mutable_index_type<transaction_index>();
    const chain_id_type& chain_id = get_chain_id();
-   transaction_id_type trx_id;
    if( !(skip & skip_transaction_dupe_check) )
-   {
-      trx_id = trx.id();
-      FC_ASSERT( trx_idx.indices().get<by_trx_id>().find(trx_id) == trx_idx.indices().get<by_trx_id>().end() );
-   }
+      FC_ASSERT( trx_idx.indices().get<by_trx_id>().find(trx.id()) == trx_idx.indices().get<by_trx_id>().end() );
    transaction_evaluation_state eval_state(this);
    const chain_parameters& chain_parameters = get_global_properties().parameters;
    eval_state._trx = &trx;
@@ -667,8 +663,8 @@ processed_transaction database::_apply_transaction(const signed_transaction& trx
    //Insert transaction into unique transactions database.
    if( !(skip & skip_transaction_dupe_check) )
    {
-      create<transaction_object>([&trx_id,&trx](transaction_object& transaction) {
-         transaction.trx_id = trx_id;
+      create<transaction_object>([&trx](transaction_object& transaction) {
+         transaction.trx_id = trx.id();
          transaction.trx = trx;
       });
    }

From cb2244f3db24f23770e888657b8cb084da2d99c2 Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Mon, 8 Oct 2018 16:36:04 +0200
Subject: [PATCH 25/38] Changed push_transaction to accept
 precomputable_transaction instead of signed_transaction

---
 libraries/chain/db_block.cpp                  |  4 +-
 .../chain/include/graphene/chain/database.hpp |  6 +-
 .../chain/include/graphene/chain/db_with.hpp  | 14 +----
 tests/app/main.cpp                            |  2 +-
 tests/common/database_fixture.cpp             | 57 ++++++++++---------
 tests/tests/authority_tests.cpp               |  2 +-
 tests/tests/block_tests.cpp                   | 26 ++++-----
 tests/tests/confidential_tests.cpp            |  8 +--
 tests/tests/operation_tests.cpp               |  6 +-
 tests/tests/operation_tests2.cpp              | 36 ++++++------
 10 files changed, 78 insertions(+), 83 deletions(-)

diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp
index 4873c65ed5..ada5167ba2 100644
--- a/libraries/chain/db_block.cpp
+++ b/libraries/chain/db_block.cpp
@@ -228,7 +228,7 @@ bool database::_push_block(const signed_block& new_block)
  * queues full as well, it will be kept in the queue to be propagated later when a new block flushes out the pending
  * queues.
  */
-processed_transaction database::push_transaction( const signed_transaction& trx, uint32_t skip )
+processed_transaction database::push_transaction( const precomputable_transaction& trx, uint32_t skip )
 { try {
    processed_transaction result;
    detail::with_skip_flags( *this, skip, [&]()
@@ -238,7 +238,7 @@ processed_transaction database::push_transaction( const signed_transaction& trx,
    return result;
 } FC_CAPTURE_AND_RETHROW( (trx) ) }
 
-processed_transaction database::_push_transaction( const signed_transaction& trx )
+processed_transaction database::_push_transaction( const precomputable_transaction& trx )
 {
    // If this is the first transaction pushed after applying a block, start a new undo session.
    // This allows us to quickly rewind to the clean state of the head block, in case a new block arrives.
diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp
index db2f38e5de..bf2795ea1f 100644
--- a/libraries/chain/include/graphene/chain/database.hpp
+++ b/libraries/chain/include/graphene/chain/database.hpp
@@ -135,9 +135,9 @@ namespace graphene { namespace chain {
          bool before_last_checkpoint()const;
 
          bool push_block( const signed_block& b, uint32_t skip = skip_nothing );
-         processed_transaction push_transaction( const signed_transaction& trx, uint32_t skip = skip_nothing );
+         processed_transaction push_transaction( const precomputable_transaction& trx, uint32_t skip = skip_nothing );
          bool _push_block( const signed_block& b );
-         processed_transaction _push_transaction( const signed_transaction& trx );
+         processed_transaction _push_transaction( const precomputable_transaction& trx );
 
          ///@throws fc::exception if the proposed transaction fails to apply.
          processed_transaction push_proposal( const proposal_object& proposal );
@@ -406,7 +406,7 @@ namespace graphene { namespace chain {
 
          /** when popping a block, the transactions that were removed get cached here so they
           * can be reapplied at the proper time */
-         std::deque< signed_transaction >       _popped_tx;
+         std::deque< precomputable_transaction > _popped_tx;
 
          /**
           * @}
diff --git a/libraries/chain/include/graphene/chain/db_with.hpp b/libraries/chain/include/graphene/chain/db_with.hpp
index de93bb15f5..7ae189216f 100644
--- a/libraries/chain/include/graphene/chain/db_with.hpp
+++ b/libraries/chain/include/graphene/chain/db_with.hpp
@@ -80,11 +80,9 @@ struct pending_transactions_restorer
       {
          try {
             if( !_db.is_known_transaction( tx.id() ) ) {
-               // since push_transaction() takes a signed_transaction,
-               // the operation_results field will be ignored.
                _db._push_transaction( tx );
             }
-         } catch ( const fc::exception&  ) {
+         } catch ( const fc::exception& ) { // ignore invalid transactions
          }
       }
       _db._popped_tx.clear();
@@ -93,17 +91,11 @@ struct pending_transactions_restorer
          try
          {
             if( !_db.is_known_transaction( tx.id() ) ) {
-               // since push_transaction() takes a signed_transaction,
-               // the operation_results field will be ignored.
                _db._push_transaction( tx );
             }
          }
-         catch( const fc::exception& e )
-         {
-            /*
-            wlog( "Pending transaction became invalid after switching to block ${b}  ${t}", ("b", _db.head_block_id())("t",_db.head_block_time()) );
-            wlog( "The invalid pending transaction caused exception ${e}", ("e", e.to_detail_string() ) );
-            */
+         catch( const fc::exception& )
+         { // ignore invalid transactions
          }
       }
    }
diff --git a/tests/app/main.cpp b/tests/app/main.cpp
index 848d60d664..b68cdf78a0 100644
--- a/tests/app/main.cpp
+++ b/tests/app/main.cpp
@@ -267,7 +267,7 @@ BOOST_AUTO_TEST_CASE( two_node_network )
       BOOST_CHECK_EQUAL( db2->get_balance( GRAPHENE_NULL_ACCOUNT, asset_id_type() ).amount.value, 0 );
 
       BOOST_TEST_MESSAGE( "Creating transfer tx" );
-      graphene::chain::signed_transaction trx;
+      graphene::chain::precomputable_transaction trx;
       {
          account_id_type nathan_id = db2->get_index_type<account_index>().indices().get<by_name>().find( "nathan" )->id;
          fc::ecc::private_key nathan_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("nathan")));
diff --git a/tests/common/database_fixture.cpp b/tests/common/database_fixture.cpp
index 2e05f71017..874e2f8241 100644
--- a/tests/common/database_fixture.cpp
+++ b/tests/common/database_fixture.cpp
@@ -30,6 +30,7 @@
 #include <graphene/elasticsearch/elasticsearch_plugin.hpp>
 #include <graphene/es_objects/es_objects.hpp>
 
+#include <graphene/chain/balance_object.hpp>
 #include <graphene/chain/committee_member_object.hpp>
 #include <graphene/chain/fba_object.hpp>
 #include <graphene/chain/market_object.hpp>
@@ -234,7 +235,7 @@ void database_fixture::verify_asset_supplies( const database& db )
    BOOST_CHECK(core_asset_data.fee_pool == 0);
 
    const auto& statistics_index = db.get_index_type<account_stats_index>().indices();
-   const auto& balance_index = db.get_index_type<account_balance_index>().indices();
+   const auto& acct_balance_index = db.get_index_type<account_balance_index>().indices();
    const auto& settle_index = db.get_index_type<force_settlement_index>().indices();
    const auto& bids = db.get_index_type<collateral_bid_index>().indices();
    map<asset_id_type,share_type> total_balances;
@@ -242,7 +243,7 @@ void database_fixture::verify_asset_supplies( const database& db )
    share_type core_in_orders;
    share_type reported_core_in_orders;
 
-   for( const account_balance_object& b : balance_index )
+   for( const account_balance_object& b : acct_balance_index )
       total_balances[b.asset_type] += b.balance;
    for( const force_settlement_object& s : settle_index )
       total_balances[s.balance.asset_id] += s.balance.amount;
@@ -284,6 +285,8 @@ void database_fixture::verify_asset_supplies( const database& db )
       total_balances[ vbo.balance.asset_id ] += vbo.balance.amount;
    for( const fba_accumulator_object& fba : db.get_index_type< simple_index< fba_accumulator_object > >() )
       total_balances[ asset_id_type() ] += fba.accumulated_fba_fees;
+   for( const balance_object& bo : db.get_index_type< balance_index >().indices() )
+      total_balances[ bo.balance.asset_id ] += bo.balance.amount;
 
    total_balances[asset_id_type()] += db.get_dynamic_global_properties().witness_budget;
 
@@ -460,7 +463,7 @@ const asset_object& database_fixture::create_bitasset(
    creator.bitasset_opts->short_backing_asset = backing_asset;
    trx.operations.push_back(std::move(creator));
    trx.validate();
-   processed_transaction ptx = db.push_transaction(trx, ~0);
+   processed_transaction ptx = PUSH_TX(db, trx, ~0);
    trx.operations.clear();
    return db.get<asset_object>(ptx.operation_results[0].get<object_id_type>());
 } FC_CAPTURE_AND_RETHROW( (name)(flags) ) }
@@ -491,7 +494,7 @@ const asset_object& database_fixture::create_prediction_market(
    creator.is_prediction_market = true;
    trx.operations.push_back(std::move(creator));
    trx.validate();
-   processed_transaction ptx = db.push_transaction(trx, ~0);
+   processed_transaction ptx = PUSH_TX(db, trx, ~0);
    trx.operations.clear();
    return db.get<asset_object>(ptx.operation_results[0].get<object_id_type>());
 } FC_CAPTURE_AND_RETHROW( (name)(flags) ) }
@@ -511,7 +514,7 @@ const asset_object& database_fixture::create_user_issued_asset( const string& na
    creator.common_options.issuer_permissions = charge_market_fee;
    trx.operations.push_back(std::move(creator));
    trx.validate();
-   processed_transaction ptx = db.push_transaction(trx, ~0);
+   processed_transaction ptx = PUSH_TX(db, trx, ~0);
    trx.operations.clear();
    return db.get<asset_object>(ptx.operation_results[0].get<object_id_type>());
 }
@@ -533,7 +536,7 @@ const asset_object& database_fixture::create_user_issued_asset( const string& na
    trx.operations.push_back(std::move(creator));
    set_expiration( db, trx );
    trx.validate();
-   processed_transaction ptx = db.push_transaction(trx, ~0);
+   processed_transaction ptx = PUSH_TX(db, trx, ~0);
    trx.operations.clear();
    return db.get<asset_object>(ptx.operation_results[0].get<object_id_type>());
 }
@@ -546,7 +549,7 @@ void database_fixture::issue_uia( const account_object& recipient, asset amount
    op.asset_to_issue = amount;
    op.issue_to_account = recipient.id;
    trx.operations.push_back(op);
-   db.push_transaction( trx, ~0 );
+   PUSH_TX( db, trx, ~0 );
    trx.operations.clear();
 }
 
@@ -592,7 +595,7 @@ const account_object& database_fixture::create_account(
 {
    trx.operations.push_back(make_account(name, key));
    trx.validate();
-   processed_transaction ptx = db.push_transaction(trx, ~0);
+   processed_transaction ptx = PUSH_TX(db, trx, ~0);
    auto& result = db.get<account_object>(ptx.operation_results[0].get<object_id_type>());
    trx.operations.clear();
    return result;
@@ -611,7 +614,7 @@ const account_object& database_fixture::create_account(
       trx.operations.resize(1);
       trx.operations.back() = (make_account(name, registrar, referrer, referrer_percent, key));
       trx.validate();
-      auto r = db.push_transaction(trx, ~0);
+      auto r = PUSH_TX(db, trx, ~0);
       const auto& result = db.get<account_object>(r.operation_results[0].get<object_id_type>());
       trx.operations.clear();
       return result;
@@ -643,7 +646,7 @@ const account_object& database_fixture::create_account(
 
       trx.validate();
 
-      processed_transaction ptx = db.push_transaction(trx, ~0);
+      processed_transaction ptx = PUSH_TX(db, trx, ~0);
       const account_object& result = db.get<account_object>(ptx.operation_results[0].get<object_id_type>());
       trx.operations.clear();
       return result;
@@ -657,7 +660,7 @@ const committee_member_object& database_fixture::create_committee_member( const
    op.committee_member_account = owner.id;
    trx.operations.push_back(op);
    trx.validate();
-   processed_transaction ptx = db.push_transaction(trx, ~0);
+   processed_transaction ptx = PUSH_TX(db, trx, ~0);
    trx.operations.clear();
    return db.get<committee_member_object>(ptx.operation_results[0].get<object_id_type>());
 }
@@ -678,7 +681,7 @@ const witness_object& database_fixture::create_witness( const account_object& ow
    op.block_signing_key = signing_private_key.get_public_key();
    trx.operations.push_back(op);
    trx.validate();
-   processed_transaction ptx = db.push_transaction(trx, skip_flags );
+   processed_transaction ptx = PUSH_TX(db, trx, skip_flags );
    trx.clear();
    return db.get<witness_object>(ptx.operation_results[0].get<object_id_type>());
 } FC_CAPTURE_AND_RETHROW() }
@@ -693,7 +696,7 @@ const worker_object& database_fixture::create_worker( const account_id_type owne
    op.work_end_date = op.work_begin_date + duration;
    trx.operations.push_back(op);
    trx.validate();
-   processed_transaction ptx = db.push_transaction(trx, ~0);
+   processed_transaction ptx = PUSH_TX(db, trx, ~0);
    trx.clear();
    return db.get<worker_object>(ptx.operation_results[0].get<object_id_type>());
 } FC_CAPTURE_AND_RETHROW() }
@@ -738,7 +741,7 @@ const limit_order_object* database_fixture::create_sell_order( const account_obj
    trx.operations.push_back(buy_order);
    for( auto& op : trx.operations ) db.current_fee_schedule().set_fee(op, fee_core_exchange_rate);
    trx.validate();
-   auto processed = db.push_transaction(trx, ~0);
+   auto processed = PUSH_TX(db, trx, ~0);
    trx.operations.clear();
    verify_asset_supplies(db);
    return db.find<limit_order_object>( processed.operation_results[0].get<object_id_type>() );
@@ -752,7 +755,7 @@ asset database_fixture::cancel_limit_order( const limit_order_object& order )
   trx.operations.push_back(cancel_order);
   for( auto& op : trx.operations ) db.current_fee_schedule().set_fee(op);
   trx.validate();
-  auto processed = db.push_transaction(trx, ~0);
+  auto processed = PUSH_TX(db, trx, ~0);
   trx.operations.clear();
    verify_asset_supplies(db);
   return processed.operation_results[0].get<asset>();
@@ -788,7 +791,7 @@ void database_fixture::transfer(
          for( auto& op : trx.operations ) db.current_fee_schedule().set_fee(op);
       }
       trx.validate();
-      db.push_transaction(trx, ~0);
+      PUSH_TX(db, trx, ~0);
       verify_asset_supplies(db);
       trx.operations.clear();
    } FC_CAPTURE_AND_RETHROW( (from.id)(to.id)(amount)(fee) )
@@ -806,7 +809,7 @@ void database_fixture::update_feed_producers( const asset_object& mia, flat_set<
 
    for( auto& op : trx.operations ) db.current_fee_schedule().set_fee(op);
    trx.validate();
-   db.push_transaction(trx, ~0);
+   PUSH_TX(db, trx, ~0);
    trx.operations.clear();
    verify_asset_supplies(db);
 } FC_CAPTURE_AND_RETHROW( (mia)(producers) ) }
@@ -826,7 +829,7 @@ void database_fixture::publish_feed( const asset_object& mia, const account_obje
 
    for( auto& op : trx.operations ) db.current_fee_schedule().set_fee(op);
    trx.validate();
-   db.push_transaction(trx, ~0);
+   PUSH_TX(db, trx, ~0);
    trx.operations.clear();
    verify_asset_supplies(db);
 }
@@ -874,7 +877,7 @@ void database_fixture::force_global_settle( const asset_object& what, const pric
    trx.operations.push_back(sop);
    for( auto& op : trx.operations ) db.current_fee_schedule().set_fee(op);
    trx.validate();
-   db.push_transaction(trx, ~0);
+   PUSH_TX(db, trx, ~0);
    trx.operations.clear();
    verify_asset_supplies(db);
 } FC_CAPTURE_AND_RETHROW( (what)(p) ) }
@@ -889,7 +892,7 @@ operation_result database_fixture::force_settle( const account_object& who, asse
    trx.operations.push_back(sop);
    for( auto& op : trx.operations ) db.current_fee_schedule().set_fee(op);
    trx.validate();
-   processed_transaction ptx = db.push_transaction(trx, ~0);
+   processed_transaction ptx = PUSH_TX(db, trx, ~0);
    const operation_result& op_result = ptx.operation_results.front();
    trx.operations.clear();
    verify_asset_supplies(db);
@@ -909,7 +912,7 @@ const call_order_object* database_fixture::borrow( const account_object& who, as
    trx.operations.push_back(update);
    for( auto& op : trx.operations ) db.current_fee_schedule().set_fee(op);
    trx.validate();
-   db.push_transaction(trx, ~0);
+   PUSH_TX(db, trx, ~0);
    trx.operations.clear();
    verify_asset_supplies(db);
 
@@ -934,7 +937,7 @@ void database_fixture::cover(const account_object& who, asset what, asset collat
    trx.operations.push_back(update);
    for( auto& op : trx.operations ) db.current_fee_schedule().set_fee(op);
    trx.validate();
-   db.push_transaction(trx, ~0);
+   PUSH_TX(db, trx, ~0);
    trx.operations.clear();
    verify_asset_supplies(db);
 } FC_CAPTURE_AND_RETHROW( (who.name)(what)(collateral)(target_cr) ) }
@@ -950,7 +953,7 @@ void database_fixture::bid_collateral(const account_object& who, const asset& to
    trx.operations.push_back(bid);
    for( auto& op : trx.operations ) db.current_fee_schedule().set_fee(op);
    trx.validate();
-   db.push_transaction(trx, ~0);
+   PUSH_TX(db, trx, ~0);
    trx.operations.clear();
    verify_asset_supplies(db);
 } FC_CAPTURE_AND_RETHROW( (who.name)(to_bid)(to_cover) ) }
@@ -966,7 +969,7 @@ void database_fixture::fund_fee_pool( const account_object& from, const asset_ob
    for( auto& op : trx.operations ) db.current_fee_schedule().set_fee(op);
    trx.validate();
    set_expiration( db, trx );
-   db.push_transaction(trx, ~0);
+   PUSH_TX(db, trx, ~0);
    trx.operations.clear();
    verify_asset_supplies(db);
 }
@@ -993,7 +996,7 @@ void database_fixture::upgrade_to_lifetime_member( const account_object& account
       op.upgrade_to_lifetime_member = true;
       op.fee = db.get_global_properties().parameters.current_fees->calculate_fee(op);
       trx.operations = {op};
-      db.push_transaction(trx, ~0);
+      PUSH_TX(db, trx, ~0);
       FC_ASSERT( op.account_to_upgrade(db).is_lifetime_member() );
       trx.clear();
       verify_asset_supplies(db);
@@ -1013,7 +1016,7 @@ void database_fixture::upgrade_to_annual_member(const account_object& account)
       op.account_to_upgrade = account.get_id();
       op.fee = db.get_global_properties().parameters.current_fees->calculate_fee(op);
       trx.operations = {op};
-      db.push_transaction(trx, ~0);
+      PUSH_TX(db, trx, ~0);
       FC_ASSERT( op.account_to_upgrade(db).is_member(db.head_block_time()) );
       trx.clear();
       verify_asset_supplies(db);
@@ -1180,7 +1183,7 @@ bool _push_block( database& db, const signed_block& b, uint32_t skip_flags /* =
 
 processed_transaction _push_transaction( database& db, const signed_transaction& tx, uint32_t skip_flags /* = 0 */ )
 { try {
-   auto pt = db.push_transaction( tx, skip_flags );
+   auto pt = db.push_transaction( precomputable_transaction(tx), skip_flags );
    database_fixture::verify_asset_supplies(db);
    return pt;
 } FC_CAPTURE_AND_RETHROW((tx)) }
diff --git a/tests/tests/authority_tests.cpp b/tests/tests/authority_tests.cpp
index 573d609edf..37bfb8f7cf 100644
--- a/tests/tests/authority_tests.cpp
+++ b/tests/tests/authority_tests.cpp
@@ -498,7 +498,7 @@ BOOST_AUTO_TEST_CASE( committee_authority )
    */
    trx.operations.push_back(uop);
    sign( trx, committee_key );
-   db.push_transaction(trx);
+   PUSH_TX(db, trx);
    BOOST_CHECK_EQUAL(get_balance(nathan, asset_id_type()(db)), 0);
    BOOST_CHECK(db.get<proposal_object>(prop.id).is_authorized_to_execute(db));
 
diff --git a/tests/tests/block_tests.cpp b/tests/tests/block_tests.cpp
index 1890dc38b8..837a1d9bb2 100644
--- a/tests/tests/block_tests.cpp
+++ b/tests/tests/block_tests.cpp
@@ -602,11 +602,11 @@ BOOST_AUTO_TEST_CASE( undo_pending )
          t.to = nathan_id;
          t.amount = asset(5000);
          trx.operations.push_back(t);
-         db.push_transaction(trx, ~0);
+         PUSH_TX(db, trx, ~0);
          trx.clear();
          set_expiration( db, trx );
          trx.operations.push_back(t);
-         db.push_transaction(trx, ~0);
+         PUSH_TX(db, trx, ~0);
 
          BOOST_CHECK(db.get_balance(nathan_id, asset_id_type()).amount == 10000);
          db.clear_pending();
@@ -757,7 +757,7 @@ BOOST_AUTO_TEST_CASE( tapos )
       cop.active = cop.owner;
       trx.operations.push_back(cop);
       trx.sign( init_account_priv_key, db1.get_chain_id() );
-      db1.push_transaction(trx);
+      PUSH_TX(db1, trx);
       b = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing);
       trx.clear();
 
@@ -952,7 +952,7 @@ BOOST_FIXTURE_TEST_CASE( double_sign_check, database_fixture )
    for( auto& op : trx.operations ) db.current_fee_schedule().set_fee(op);
    trx.validate();
 
-   db.push_transaction(trx, ~0);
+   PUSH_TX(db, trx, ~0);
 
    trx.operations.clear();
    t.from = bob.id;
@@ -963,21 +963,21 @@ BOOST_FIXTURE_TEST_CASE( double_sign_check, database_fixture )
    trx.validate();
 
    BOOST_TEST_MESSAGE( "Verify that not-signing causes an exception" );
-   GRAPHENE_REQUIRE_THROW( db.push_transaction(trx, 0), fc::exception );
+   GRAPHENE_REQUIRE_THROW( PUSH_TX(db, trx, 0), fc::exception );
 
    BOOST_TEST_MESSAGE( "Verify that double-signing causes an exception" );
    sign( trx, bob_private_key );
    sign( trx, bob_private_key );
-   GRAPHENE_REQUIRE_THROW( db.push_transaction(trx, 0), tx_duplicate_sig );
+   GRAPHENE_REQUIRE_THROW( PUSH_TX(db, trx, 0), tx_duplicate_sig );
 
    BOOST_TEST_MESSAGE( "Verify that signing with an extra, unused key fails" );
    trx.signatures.pop_back();
    sign( trx, generate_private_key("bogus" ));
-   GRAPHENE_REQUIRE_THROW( db.push_transaction(trx, 0), tx_irrelevant_sig );
+   GRAPHENE_REQUIRE_THROW( PUSH_TX(db, trx, 0), tx_irrelevant_sig );
 
    BOOST_TEST_MESSAGE( "Verify that signing once with the proper key passes" );
    trx.signatures.pop_back();
-   db.push_transaction(trx, 0);
+   PUSH_TX(db, trx, 0);
 
 } FC_LOG_AND_RETHROW() }
 
@@ -998,7 +998,7 @@ BOOST_FIXTURE_TEST_CASE( change_block_interval, database_fixture )
       uop.new_parameters.block_interval = 1;
       cop.proposed_ops.emplace_back(uop);
       trx.operations.push_back(cop);
-      db.push_transaction(trx);
+      PUSH_TX(db, trx);
    }
    BOOST_TEST_MESSAGE( "Updating proposal by signing with the committee_member private key" );
    {
@@ -1019,7 +1019,7 @@ BOOST_FIXTURE_TEST_CASE( change_block_interval, database_fixture )
       sign( trx, get_account("init6" ).active.get_keys().front(),init_account_priv_key);
       sign( trx, get_account("init7" ).active.get_keys().front(),init_account_priv_key);
       */
-      db.push_transaction(trx);
+      PUSH_TX(db, trx);
       BOOST_CHECK(proposal_id_type()(db).is_authorized_to_execute(db));
    }
    BOOST_TEST_MESSAGE( "Verifying that the interval didn't change immediately" );
@@ -1602,7 +1602,7 @@ BOOST_FIXTURE_TEST_CASE( update_account_keys, database_fixture )
                trx.operations.push_back( create_op );
                // trx.sign( sam_key );
 
-               processed_transaction ptx_create = db.push_transaction( trx,
+               processed_transaction ptx_create = PUSH_TX( db, trx,
                   database::skip_transaction_dupe_check |
                   database::skip_transaction_signatures
                );
@@ -1639,11 +1639,11 @@ BOOST_FIXTURE_TEST_CASE( update_account_keys, database_fixture )
                      sign( trx, *owner_privkey[i] );
                      if( i < int(create_op.owner.weight_threshold-1) )
                      {
-                        GRAPHENE_REQUIRE_THROW(db.push_transaction(trx), fc::exception);
+                        GRAPHENE_REQUIRE_THROW(PUSH_TX(db, trx), fc::exception);
                      }
                      else
                      {
-                        db.push_transaction( trx,
+                        PUSH_TX( db, trx,
                            database::skip_transaction_dupe_check |
                            database::skip_transaction_signatures );
                      }
diff --git a/tests/tests/confidential_tests.cpp b/tests/tests/confidential_tests.cpp
index bef651db5c..b5046dcdfe 100644
--- a/tests/tests/confidential_tests.cpp
+++ b/tests/tests/confidential_tests.cpp
@@ -77,7 +77,7 @@ BOOST_AUTO_TEST_CASE( confidential_test )
 
    trx.operations = {to_blind};
    sign( trx,  dan_private_key  );
-   db.push_transaction(trx);
+   PUSH_TX(db, trx);
    trx.clear_signatures();
 
    BOOST_TEST_MESSAGE( "Transfering from blind to blind with change address" );
@@ -97,7 +97,7 @@ BOOST_AUTO_TEST_CASE( confidential_test )
    blind_tr.validate();
    trx.operations = {blind_tr};
    sign( trx,  owner2_key  );
-   db.push_transaction(trx);
+   PUSH_TX(db, trx);
 
    BOOST_TEST_MESSAGE( "Attempting to double spend the same commitments" );
    blind_tr.fee = core.amount(11);
@@ -108,7 +108,7 @@ BOOST_AUTO_TEST_CASE( confidential_test )
    out4.range_proof = fc::ecc::range_proof_sign( 0, out3.commitment, InB1, nonce1, 0, 0, 750-300-11 );
    blind_tr.outputs = {out4,out3};
    trx.operations = {blind_tr};
-   BOOST_REQUIRE_THROW( db.push_transaction(trx, ~0), graphene::chain::blind_transfer_unknown_commitment );
+   BOOST_REQUIRE_THROW( PUSH_TX(db, trx, ~0), graphene::chain::blind_transfer_unknown_commitment );
 
 
    BOOST_TEST_MESSAGE( "Transfering from blind to nathan public" );
@@ -122,7 +122,7 @@ BOOST_AUTO_TEST_CASE( confidential_test )
    from_blind.inputs.push_back( {out4.commitment, out4.owner} );
    trx.operations = {from_blind};
    trx.clear_signatures();
-   db.push_transaction(trx);
+   PUSH_TX(db, trx);
 
    BOOST_REQUIRE_EQUAL( get_balance( nathan, core ), 750-300-10-10 );
 
diff --git a/tests/tests/operation_tests.cpp b/tests/tests/operation_tests.cpp
index b00745ff06..398eb42b11 100644
--- a/tests/tests/operation_tests.cpp
+++ b/tests/tests/operation_tests.cpp
@@ -303,7 +303,7 @@ BOOST_AUTO_TEST_CASE( asset_settle_cancel_operation_test_after_hf588 )
       pcop.proposed_ops.emplace_back(ascop);
       trx.operations.push_back(pcop);
 
-      BOOST_CHECK_EXCEPTION(db.push_transaction(trx), fc::assert_exception,
+      BOOST_CHECK_EXCEPTION(PUSH_TX(db, trx), fc::assert_exception,
             [](fc::assert_exception const &e) -> bool {
                std::cout << e.to_string() << std::endl;
                if (e.to_string().find("Virtual operation") != std::string::npos)
@@ -333,7 +333,7 @@ BOOST_AUTO_TEST_CASE( asset_settle_cancel_operation_test_after_hf588 )
 
       trx.operations.push_back(pcop);
 
-      BOOST_CHECK_EXCEPTION(db.push_transaction(trx), fc::assert_exception,
+      BOOST_CHECK_EXCEPTION(PUSH_TX(db, trx), fc::assert_exception,
             [](fc::assert_exception const &e) -> bool {
                std::cout << e.to_string() << std::endl;
                if (e.to_string().find("Virtual operation") != std::string::npos)
@@ -656,7 +656,7 @@ BOOST_AUTO_TEST_CASE( call_order_update_target_cr_hardfork_time_test )
          tx.operations.push_back( prop );
          db.current_fee_schedule().set_fee( tx.operations.back() );
          set_expiration( db, tx );
-         db.push_transaction( tx, ~0 );
+         PUSH_TX( db, tx, ~0 );
       };
 
       BOOST_TEST_MESSAGE( "bob tries to propose a proposal with target_cr set, "
diff --git a/tests/tests/operation_tests2.cpp b/tests/tests/operation_tests2.cpp
index e0d027da1c..15df1b2ece 100644
--- a/tests/tests/operation_tests2.cpp
+++ b/tests/tests/operation_tests2.cpp
@@ -139,7 +139,7 @@ BOOST_AUTO_TEST_CASE( withdraw_permission_create_before_hardfork_23 )
       trx.operations.back() = op;
    }
    sign( trx, nathan_private_key );
-   db.push_transaction( trx );
+   PUSH_TX( db, trx );
    trx.clear();
 } FC_LOG_AND_RETHROW() }
 
@@ -184,7 +184,7 @@ BOOST_AUTO_TEST_CASE( withdraw_permission_create_after_hardfork_23 )
       trx.operations.back() = op;
    }
    sign( trx, nathan_private_key );
-   db.push_transaction( trx );
+   PUSH_TX( db, trx );
    trx.clear();
 } FC_LOG_AND_RETHROW() }
 
@@ -1027,7 +1027,7 @@ BOOST_AUTO_TEST_CASE( feed_limit_test )
    op.issuer = bit_usd.issuer;
    trx.operations = {op};
    sign( trx, nathan_private_key );
-   db.push_transaction(trx);
+   PUSH_TX(db, trx);
 
    BOOST_TEST_MESSAGE("Checking current_feed is null");
    BOOST_CHECK(bitasset.current_feed.settlement_price.is_null());
@@ -1037,7 +1037,7 @@ BOOST_AUTO_TEST_CASE( feed_limit_test )
    trx.clear();
    trx.operations = {op};
    sign( trx, nathan_private_key );
-   db.push_transaction(trx);
+   PUSH_TX(db, trx);
 
    BOOST_TEST_MESSAGE("Checking current_feed is not null");
    BOOST_CHECK(!bitasset.current_feed.settlement_price.is_null());
@@ -1907,13 +1907,13 @@ BOOST_AUTO_TEST_CASE( balance_object_test )
    trx.operations = {op};
    _sign( trx, n_key );
    // Fail because I'm claiming from an address which hasn't signed
-   GRAPHENE_CHECK_THROW(db.push_transaction(trx), tx_missing_other_auth);
+   GRAPHENE_CHECK_THROW(PUSH_TX(db, trx), tx_missing_other_auth);
    trx.clear();
    op.balance_to_claim = balance_id_type();
    op.balance_owner_key = n_key.get_public_key();
    trx.operations = {op};
    _sign( trx, n_key );
-   db.push_transaction(trx);
+   PUSH_TX(db, trx);
 
    // Not using fixture's get_balance() here because it uses fixture's db, not my override
    BOOST_CHECK_EQUAL(db.get_balance(op.deposit_to_account, asset_id_type()).amount.value, 1);
@@ -1941,7 +1941,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test )
    _sign( trx, n_key );
    _sign( trx, v1_key );
    // Attempting to claim 1 from a balance with 0 available
-   GRAPHENE_CHECK_THROW(db.push_transaction(trx), balance_claim_invalid_claim_amount);
+   GRAPHENE_CHECK_THROW(PUSH_TX(db, trx), balance_claim_invalid_claim_amount);
 
    op.balance_to_claim = vesting_balance_2.id;
    op.total_claimed.amount = 151;
@@ -1951,7 +1951,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test )
    _sign( trx, n_key );
    _sign( trx, v2_key );
    // Attempting to claim 151 from a balance with 150 available
-   GRAPHENE_CHECK_THROW(db.push_transaction(trx), balance_claim_invalid_claim_amount);
+   GRAPHENE_CHECK_THROW(PUSH_TX(db, trx), balance_claim_invalid_claim_amount);
 
    op.balance_to_claim = vesting_balance_2.id;
    op.total_claimed.amount = 100;
@@ -1960,7 +1960,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test )
    trx.clear_signatures();
    _sign( trx, n_key );
    _sign( trx, v2_key );
-   db.push_transaction(trx);
+   PUSH_TX(db, trx);
    BOOST_CHECK_EQUAL(db.get_balance(op.deposit_to_account, asset_id_type()).amount.value, 101);
    BOOST_CHECK_EQUAL(vesting_balance_2.balance.amount.value, 300);
 
@@ -1970,7 +1970,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test )
    _sign( trx, n_key );
    _sign( trx, v2_key );
    // Attempting to claim twice within a day
-   GRAPHENE_CHECK_THROW(db.push_transaction(trx), balance_claim_claimed_too_often);
+   GRAPHENE_CHECK_THROW(PUSH_TX(db, trx), balance_claim_claimed_too_often);
 
    db.generate_block(db.get_slot_time(1), db.get_scheduled_witness(1), init_account_priv_key, skip_flags);
    slot = db.get_slot_at_time(vesting_balance_1.vesting_policy->begin_timestamp + 60);
@@ -1984,7 +1984,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test )
    trx.clear_signatures();
    _sign( trx, n_key );
    _sign( trx, v1_key );
-   db.push_transaction(trx);
+   PUSH_TX(db, trx);
    BOOST_CHECK(db.find_object(op.balance_to_claim) == nullptr);
    BOOST_CHECK_EQUAL(db.get_balance(op.deposit_to_account, asset_id_type()).amount.value, 601);
 
@@ -1996,7 +1996,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test )
    _sign( trx, n_key );
    _sign( trx, v2_key );
    // Attempting to claim twice within a day
-   GRAPHENE_CHECK_THROW(db.push_transaction(trx), balance_claim_claimed_too_often);
+   GRAPHENE_CHECK_THROW(PUSH_TX(db, trx), balance_claim_claimed_too_often);
 
    db.generate_block(db.get_slot_time(1), db.get_scheduled_witness(1), init_account_priv_key, skip_flags);
    slot = db.get_slot_at_time(db.head_block_time() + fc::days(1));
@@ -2008,7 +2008,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test )
    trx.clear_signatures();
    _sign( trx, n_key );
    _sign( trx, v2_key );
-   db.push_transaction(trx);
+   PUSH_TX(db, trx);
    BOOST_CHECK(db.find_object(op.balance_to_claim) == nullptr);
    BOOST_CHECK_EQUAL(db.get_balance(op.deposit_to_account, asset_id_type()).amount.value, 901);
 } FC_LOG_AND_RETHROW() }
@@ -2028,7 +2028,7 @@ BOOST_AUTO_TEST_CASE(transfer_with_memo) {
       op.memo->set_message(alice_private_key, bob_public_key, "Dear Bob,\n\nMoney!\n\nLove, Alice");
       trx.operations = {op};
       trx.sign(alice_private_key, db.get_chain_id());
-      db.push_transaction(trx);
+      PUSH_TX(db, trx);
 
       BOOST_CHECK_EQUAL(get_balance(alice_id, asset_id_type()), 500);
       BOOST_CHECK_EQUAL(get_balance(bob_id, asset_id_type()), 500);
@@ -2429,18 +2429,18 @@ BOOST_AUTO_TEST_CASE( buyback )
             sign( tx, philbin_private_key );
 
             // Alice and Philbin signed, but asset issuer is invalid
-            GRAPHENE_CHECK_THROW( db.push_transaction(tx), account_create_buyback_incorrect_issuer );
+            GRAPHENE_CHECK_THROW( PUSH_TX(db, tx), account_create_buyback_incorrect_issuer );
 
             tx.clear_signatures();
             tx.operations.back().get< account_create_operation >().extensions.value.buyback_options->asset_to_buy_issuer = izzy_id;
             sign( tx, philbin_private_key );
 
             // Izzy didn't sign
-            GRAPHENE_CHECK_THROW( db.push_transaction(tx), tx_missing_active_auth );
+            GRAPHENE_CHECK_THROW( PUSH_TX(db, tx), tx_missing_active_auth );
             sign( tx, izzy_private_key );
 
             // OK
-            processed_transaction ptx = db.push_transaction( tx );
+            processed_transaction ptx = PUSH_TX( db, tx );
             rex_id = ptx.operation_results.back().get< object_id_type >();
 
             // Try to create another account rex2 which is bbo on same asset
@@ -2448,7 +2448,7 @@ BOOST_AUTO_TEST_CASE( buyback )
             tx.operations.back().get< account_create_operation >().name = "rex2";
             sign( tx, izzy_private_key );
             sign( tx, philbin_private_key );
-            GRAPHENE_CHECK_THROW( db.push_transaction(tx), account_create_buyback_already_exists );
+            GRAPHENE_CHECK_THROW( PUSH_TX(db, tx), account_create_buyback_already_exists );
          }
 
          // issue some BUYME to Alice

From 883ec6a7f3ffb01fcba3130ec0f21a1d39977749 Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Thu, 11 Oct 2018 17:57:22 +0200
Subject: [PATCH 26/38] Avoid one level of indirection on precompute block

---
 libraries/chain/db_block.cpp                  | 27 +++++++------------
 .../chain/include/graphene/chain/database.hpp |  1 -
 2 files changed, 10 insertions(+), 18 deletions(-)

diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp
index ada5167ba2..b26f2380cf 100644
--- a/libraries/chain/db_block.cpp
+++ b/libraries/chain/db_block.cpp
@@ -765,18 +765,16 @@ void database::_precompute_parallel( const Trx* trx, const size_t count, const u
    }
 }
 
-void database::_precompute_parallel( const signed_block& block, const uint32_t skip )const
+fc::future<void> database::precompute_parallel( const signed_block& block, const uint32_t skip )const
 { try {
-   const bool cheap = (skip & skip_expensive) == skip_expensive;
    std::vector<fc::future<void>> workers;
    if( !block.transactions.empty() )
    {
-      if( cheap )
+      if( (skip & skip_expensive) == skip_expensive )
          _precompute_parallel( &block.transactions[0], block.transactions.size(), skip );
       else
       {
          uint32_t chunks = fc::asio::default_io_service_scope::get_num_threads();
-         if( !chunks ) return;
          uint32_t chunk_size = block.transactions.size() / chunks;
          if( chunks * chunk_size < block.transactions.size() )
             chunk_size++;
@@ -795,21 +793,16 @@ void database::_precompute_parallel( const signed_block& block, const uint32_t s
    if( !(skip&skip_merkle_check) )
       block.calculate_merkle_root();
    block.id();
-   for( auto& worker : workers )
-      worker.wait();
-} FC_LOG_AND_RETHROW() }
 
-fc::future<void> database::precompute_parallel( const signed_block& block, const uint32_t skip )const
-{
-   if( block.transactions.empty() || (skip & skip_expensive) == skip_expensive )
-   {
-      _precompute_parallel( block, skip );
+   if( workers.empty() )
       return fc::future< void >( fc::promise< void >::ptr( new fc::promise< void >( true ) ) );
-   }
-   return fc::do_parallel([this,&block,skip] () {
-      _precompute_parallel( block, skip );
-   });
-}
+
+   auto first = workers.begin();
+   auto worker = first;
+   while( ++worker != workers.end() )
+      worker->wait();
+   return *first;
+} FC_LOG_AND_RETHROW() }
 
 fc::future<void> database::precompute_parallel( const precomputable_transaction& trx )const
 {
diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp
index bf2795ea1f..84e4ad4190 100644
--- a/libraries/chain/include/graphene/chain/database.hpp
+++ b/libraries/chain/include/graphene/chain/database.hpp
@@ -437,7 +437,6 @@ namespace graphene { namespace chain {
    private:
          template<typename Trx>
          void _precompute_parallel( const Trx* trx, const size_t count, const uint32_t skip )const;
-         void _precompute_parallel( const signed_block& block, const uint32_t skip )const;
 
    protected:
          //Mark pop_undo() as protected -- we do not want outside calling pop_undo(); it should call pop_block() instead

From d032890de9044fc5e2165e6185e721db5be07f68 Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Thu, 11 Oct 2018 23:04:39 +0200
Subject: [PATCH 27/38] Make pubkey comparator generally available

---
 libraries/chain/account_object.cpp            |  6 ++---
 .../include/graphene/chain/account_object.hpp | 26 +++++++------------
 .../include/graphene/chain/protocol/types.hpp |  8 ++++++
 3 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/libraries/chain/account_object.cpp b/libraries/chain/account_object.cpp
index 952a4c7c2c..7bdf51a2de 100644
--- a/libraries/chain/account_object.cpp
+++ b/libraries/chain/account_object.cpp
@@ -121,9 +121,9 @@ set<account_id_type> account_member_index::get_account_members(const account_obj
       result.insert(auth.first);
    return result;
 }
-set<public_key_type, account_member_index::key_compare> account_member_index::get_key_members(const account_object& a)const
+set<public_key_type, pubkey_comparator> account_member_index::get_key_members(const account_object& a)const
 {
-   set<public_key_type, key_compare> result;
+   set<public_key_type, pubkey_comparator> result;
    for( auto auth : a.owner.key_auths )
       result.insert(auth.first);
    for( auto auth : a.active.key_auths )
@@ -215,7 +215,7 @@ void account_member_index::object_modified(const object& after)
 
 
     {
-       set<public_key_type, key_compare> after_key_members = get_key_members(a);
+       set<public_key_type, pubkey_comparator> after_key_members = get_key_members(a);
 
        vector<public_key_type> removed; removed.reserve(before_key_members.size());
        std::set_difference(before_key_members.begin(), before_key_members.end(),
diff --git a/libraries/chain/include/graphene/chain/account_object.hpp b/libraries/chain/include/graphene/chain/account_object.hpp
index cae9d35984..c3fdcee01f 100644
--- a/libraries/chain/include/graphene/chain/account_object.hpp
+++ b/libraries/chain/include/graphene/chain/account_object.hpp
@@ -292,14 +292,6 @@ namespace graphene { namespace chain {
     */
    class account_member_index : public secondary_index
    {
-      class key_compare {
-      public:
-         inline bool operator()( const public_key_type& a, const public_key_type& b )const
-         {
-            return a.key_data < b.key_data;
-         }
-      };
-
       public:
          virtual void object_inserted( const object& obj ) override;
          virtual void object_removed( const object& obj ) override;
@@ -308,20 +300,20 @@ namespace graphene { namespace chain {
 
 
          /** given an account or key, map it to the set of accounts that reference it in an active or owner authority */
-         map< account_id_type, set<account_id_type> > account_to_account_memberships;
-         map< public_key_type, set<account_id_type>, key_compare > account_to_key_memberships;
+         map< account_id_type, set<account_id_type> >                    account_to_account_memberships;
+         map< public_key_type, set<account_id_type>, pubkey_comparator > account_to_key_memberships;
          /** some accounts use address authorities in the genesis block */
-         map< address, set<account_id_type> >         account_to_address_memberships;
+         map< address, set<account_id_type> >                            account_to_address_memberships;
 
 
       protected:
-         set<account_id_type>  get_account_members( const account_object& a )const;
-         set<public_key_type, key_compare>  get_key_members( const account_object& a )const;
-         set<address>          get_address_members( const account_object& a )const;
+         set<account_id_type>                    get_account_members( const account_object& a )const;
+         set<public_key_type, pubkey_comparator> get_key_members( const account_object& a )const;
+         set<address>                            get_address_members( const account_object& a )const;
 
-         set<account_id_type>  before_account_members;
-         set<public_key_type, key_compare>  before_key_members;
-         set<address>          before_address_members;
+         set<account_id_type>                    before_account_members;
+         set<public_key_type, pubkey_comparator> before_key_members;
+         set<address>                            before_address_members;
    };
 
 
diff --git a/libraries/chain/include/graphene/chain/protocol/types.hpp b/libraries/chain/include/graphene/chain/protocol/types.hpp
index bd04c071d3..4456d6d3a5 100644
--- a/libraries/chain/include/graphene/chain/protocol/types.hpp
+++ b/libraries/chain/include/graphene/chain/protocol/types.hpp
@@ -266,6 +266,14 @@ namespace graphene { namespace chain {
        friend bool operator != ( const public_key_type& p1, const public_key_type& p2);
    };
 
+   class pubkey_comparator {
+   public:
+      inline bool operator()( const public_key_type& a, const public_key_type& b )const
+      {
+         return a.key_data < b.key_data;
+      }
+   };
+
    struct extended_public_key_type
    {
       struct binary_key

From 2c01109c6479713cd8b0f0b01346c4ce2bb91e9a Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Wed, 7 Nov 2018 17:27:47 +0100
Subject: [PATCH 28/38] Address some review comments

---
 libraries/chain/db_block.cpp                              | 4 +---
 .../chain/include/graphene/chain/protocol/transaction.hpp | 8 ++++----
 libraries/chain/protocol/transaction.cpp                  | 4 ++--
 libraries/plugins/delayed_node/delayed_node_plugin.cpp    | 1 +
 tests/common/database_fixture.hpp                         | 1 +
 5 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp
index b26f2380cf..9e0ff2c95b 100644
--- a/libraries/chain/db_block.cpp
+++ b/libraries/chain/db_block.cpp
@@ -775,9 +775,7 @@ fc::future<void> database::precompute_parallel( const signed_block& block, const
       else
       {
          uint32_t chunks = fc::asio::default_io_service_scope::get_num_threads();
-         uint32_t chunk_size = block.transactions.size() / chunks;
-         if( chunks * chunk_size < block.transactions.size() )
-            chunk_size++;
+         uint32_t chunk_size = ( block.transactions.size() + chunks - 1 ) / chunks;
          workers.reserve( chunks + 1 );
          for( size_t base = 0; base < block.transactions.size(); base += chunk_size )
             workers.push_back( fc::do_parallel( [this,&block,base,chunk_size,skip] () {
diff --git a/libraries/chain/include/graphene/chain/protocol/transaction.hpp b/libraries/chain/include/graphene/chain/protocol/transaction.hpp
index 02f00799c6..84234afb9e 100644
--- a/libraries/chain/include/graphene/chain/protocol/transaction.hpp
+++ b/libraries/chain/include/graphene/chain/protocol/transaction.hpp
@@ -89,7 +89,6 @@ namespace graphene { namespace chain {
       digest_type                        digest()const;
       virtual const transaction_id_type& id()const;
       virtual void                       validate() const;
-      /// Calculate the digest used for signature validation
 
       void set_expiration( fc::time_point_sec expiration_time );
       void set_reference_block( const block_id_type& reference_block );
@@ -115,6 +114,7 @@ namespace graphene { namespace chain {
       void get_required_authorities( flat_set<account_id_type>& active, flat_set<account_id_type>& owner, vector<authority>& other )const;
 
    protected:
+      // Calculate the digest used for signature validation
       digest_type sig_digest( const chain_id_type& chain_id )const;
       mutable transaction_id_type _tx_id_buffer;
    };
@@ -206,9 +206,9 @@ namespace graphene { namespace chain {
       precomputable_transaction( const signed_transaction& tx ) : signed_transaction(tx) {};
       precomputable_transaction( signed_transaction&& tx ) : signed_transaction( std::move(tx) ) {};
 
-      virtual const transaction_id_type&       id()const;
-      virtual void                             validate() const;
-      virtual const flat_set<public_key_type>& get_signature_keys( const chain_id_type& chain_id )const;
+      virtual const transaction_id_type&       id()const override;
+      virtual void                             validate()const override;
+      virtual const flat_set<public_key_type>& get_signature_keys( const chain_id_type& chain_id )const override;
    protected:
       mutable bool _validated = false;
    };
diff --git a/libraries/chain/protocol/transaction.cpp b/libraries/chain/protocol/transaction.cpp
index c1dd817693..1a1293ca76 100644
--- a/libraries/chain/protocol/transaction.cpp
+++ b/libraries/chain/protocol/transaction.cpp
@@ -303,8 +303,6 @@ void verify_authority( const vector<operation>& ops, const flat_set<public_key_t
 
 const flat_set<public_key_type>& signed_transaction::get_signature_keys( const chain_id_type& chain_id )const
 { try {
-   // Strictly we should check whether the given chain ID is same as the one used to initialize the `signees` field.
-   // However, we don't pass in another chain ID so far, for better performance, we skip the check.
    auto d = sig_digest( chain_id );
    flat_set<public_key_type> result;
    for( const auto&  sig : signatures )
@@ -397,6 +395,8 @@ void precomputable_transaction::validate() const
 
 const flat_set<public_key_type>& precomputable_transaction::get_signature_keys( const chain_id_type& chain_id )const
 {
+   // Strictly we should check whether the given chain ID is same as the one used to initialize the `signees` field.
+   // However, we don't pass in another chain ID so far, for better performance, we skip the check.
    if( _signees.empty() )
       signed_transaction::get_signature_keys( chain_id );
    return _signees;
diff --git a/libraries/plugins/delayed_node/delayed_node_plugin.cpp b/libraries/plugins/delayed_node/delayed_node_plugin.cpp
index dd1d4705e6..f5e3f88cd9 100644
--- a/libraries/plugins/delayed_node/delayed_node_plugin.cpp
+++ b/libraries/plugins/delayed_node/delayed_node_plugin.cpp
@@ -103,6 +103,7 @@ void delayed_node_plugin::sync_with_trusted_node()
       while( remote_dpo.last_irreversible_block_num > db.head_block_num() )
       {
          fc::optional<graphene::chain::signed_block> block = my->database_api->get_block( db.head_block_num()+1 );
+         // TODO: during sync, decouple requesting blocks from preprocessing + applying them
          FC_ASSERT(block, "Trusted node claims it has blocks it doesn't actually have.");
          ilog("Pushing block #${n}", ("n", block->block_num()));
          db.precompute_parallel( *block, graphene::chain::database::skip_nothing ).wait();
diff --git a/tests/common/database_fixture.hpp b/tests/common/database_fixture.hpp
index 72744ff7e5..eceba6876b 100644
--- a/tests/common/database_fixture.hpp
+++ b/tests/common/database_fixture.hpp
@@ -172,6 +172,7 @@ namespace graphene { namespace chain {
 
 class clearable_block : public signed_block {
 public:
+   /** @brief Clears internal cached values like ID, signing key, Merkle root etc. */
    void clear();
 };
 

From c4b584c089d515ae81d00d3fed050c5daab24520 Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Sun, 9 Dec 2018 13:43:17 +0100
Subject: [PATCH 29/38] direct_index: Added clarifying comment, ensure that
 object ID is not modified

---
 libraries/db/include/graphene/db/index.hpp | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/libraries/db/include/graphene/db/index.hpp b/libraries/db/include/graphene/db/index.hpp
index fd2923f599..ddee3833ed 100644
--- a/libraries/db/include/graphene/db/index.hpp
+++ b/libraries/db/include/graphene/db/index.hpp
@@ -194,6 +194,10 @@ namespace graphene { namespace db {
     *  @brief A secondary index that tracks objects in vectors indexed by object
     *  id. It is meant for fully (or almost fully) populated indexes only (will
     *  fail when loading an object_database with large gaps).
+    *
+    *  WARNING! If any of the methods called on insertion, removal or
+    *  modification throws, subsequent behaviour is undefined! Such exceptions
+    *  indicate that this index type is not appropriate for the use-case.
     */
    template<typename Object, uint8_t chunkbits>
    class direct_index : public secondary_index
@@ -205,6 +209,7 @@ namespace graphene { namespace db {
          static const size_t _mask = ((1 << chunkbits) - 1);
          size_t next = 0;
          vector< vector< const Object* > > content;
+         object_id_type id_being_modified;
 
       public:
          direct_index() {
@@ -254,13 +259,23 @@ namespace graphene { namespace db {
             content[instance >> chunkbits][instance & _mask] = nullptr;
          }
 
+         virtual void about_to_modify( const object& before )
+         {
+            id_being_modified = before.id;
+         }
+
+         virtual void object_modified( const object& after  )
+         {
+            FC_ASSERT( id_being_modified == after.id, "Modification of ID is not supported!");
+         }
+
          template< typename object_id >
          const Object* find( const object_id& id )const
          {
             static_assert( object_id::space_id == Object::space_id, "Space ID mismatch!" );
             static_assert( object_id::type_id == Object::type_id, "Type_ID mismatch!" );
             if( id.instance >= next ) return nullptr;
-            return content[id.instance >> chunkbits][id.instance & _mask];
+            return content[id.instance.value >> chunkbits][id.instance.value & _mask];
          };
 
          template< typename object_id >

From 9117c85dba00b6a507b9fad91cf16594423db2d5 Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Sun, 9 Dec 2018 13:43:30 +0100
Subject: [PATCH 30/38] Added unit test for direct_index

---
 tests/tests/database_tests.cpp | 75 ++++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)

diff --git a/tests/tests/database_tests.cpp b/tests/tests/database_tests.cpp
index 6b5ea8b96b..e8c1d24fe2 100644
--- a/tests/tests/database_tests.cpp
+++ b/tests/tests/database_tests.cpp
@@ -137,4 +137,79 @@ BOOST_AUTO_TEST_CASE( merge_test )
    }
 }
 
+BOOST_AUTO_TEST_CASE( direct_index_test )
+{ try {
+   try {
+      const graphene::db::primary_index< account_index, 6 > small_chunkbits( db );
+      BOOST_FAIL( "Expected assertion failure!" );
+   } catch( const fc::assert_exception& expected ) {}
+
+   graphene::db::primary_index< account_index, 8 > my_accounts( db );
+   const auto& direct = my_accounts.get_secondary_index<graphene::db::direct_index< account_object, 8 >>();
+   BOOST_CHECK_EQUAL( 0, my_accounts.indices().size() );
+   BOOST_CHECK( nullptr == direct.find( account_id_type( 1 ) ) );
+   // BOOST_CHECK_THROW( direct.find( asset_id_type( 1 ) ), fc::assert_exception ); // compile-time error
+   BOOST_CHECK_THROW( direct.find( object_id_type( asset_id_type( 1 ) ) ), fc::assert_exception );
+   BOOST_CHECK_THROW( direct.get( account_id_type( 1 ) ), fc::assert_exception );
+
+   account_object test_account;
+   test_account.id = account_id_type(1);
+   test_account.name = "account1";
+
+   my_accounts.load( fc::raw::pack( test_account ) );
+
+   BOOST_CHECK_EQUAL( 1, my_accounts.indices().size() );
+   BOOST_CHECK( nullptr == direct.find( account_id_type( 0 ) ) );
+   BOOST_CHECK( nullptr == direct.find( account_id_type( 2 ) ) );
+   BOOST_CHECK( nullptr != direct.find( account_id_type( 1 ) ) );
+   BOOST_CHECK_EQUAL( test_account.name, direct.get( test_account.id ).name );
+
+   // The following assumes that MAX_HOLE = 100
+   test_account.id = account_id_type(102);
+   test_account.name = "account102";
+   // highest insert was 1, direct.next is 2 => 102 is highest allowed instance
+   my_accounts.load( fc::raw::pack( test_account ) );
+   BOOST_CHECK_EQUAL( test_account.name, direct.get( test_account.id ).name );
+
+   // direct.next is now 103, but index sequence counter is 0
+   my_accounts.create( [] ( object& o ) {
+       account_object& acct = dynamic_cast< account_object& >( o );
+       BOOST_CHECK_EQUAL( 0, acct.id.instance() );
+       acct.name = "account0";
+   } );
+
+   test_account.id = account_id_type(50);
+   test_account.name = "account50";
+   my_accounts.load( fc::raw::pack( test_account ) );
+
+   // direct.next is still 103, so 204 is not allowed
+   test_account.id = account_id_type(204);
+   test_account.name = "account204";
+   GRAPHENE_REQUIRE_THROW( my_accounts.load( fc::raw::pack( test_account ) ), fc::assert_exception );
+   // This is actually undefined behaviour. The object has been inserted into
+   // the primary index, but the secondary has refused to insert it!
+   BOOST_CHECK_EQUAL( 5, my_accounts.indices().size() );
+
+   uint32_t count = 0;
+   for( uint32_t i = 0; i < 250; i++ )
+   {
+      const account_object* aptr = dynamic_cast< const account_object* >( my_accounts.find( account_id_type( i ) ) );
+      if( aptr )
+      {
+         count++;
+         BOOST_CHECK( aptr->id.instance() == 0 || aptr->id.instance() == 1
+                      || aptr->id.instance() == 50 || aptr->id.instance() == 102 );
+         BOOST_CHECK_EQUAL( i, aptr->id.instance() );
+         BOOST_CHECK_EQUAL( "account" + std::to_string( i ), aptr->name );
+      }
+   }
+   BOOST_CHECK_EQUAL( count, my_accounts.indices().size() - 1 );
+
+   GRAPHENE_REQUIRE_THROW( my_accounts.modify( direct.get( account_id_type( 1 ) ), [] ( object& acct ) {
+      acct.id = account_id_type(2);
+   }), fc::assert_exception );
+   // This is actually undefined behaviour. The object has been modified, but
+   // but the secondary has not updated its representation
+} FC_LOG_AND_RETHROW() }
+
 BOOST_AUTO_TEST_SUITE_END()

From 956f1c9c42869ec67ed3a1d98ad84689e4acc0d5 Mon Sep 17 00:00:00 2001
From: Alfredo <oxarbitrage@gmail.com>
Date: Fri, 14 Dec 2018 10:24:57 -0300
Subject: [PATCH 31/38] bump database

---
 libraries/chain/include/graphene/chain/config.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/chain/include/graphene/chain/config.hpp b/libraries/chain/include/graphene/chain/config.hpp
index 1ef28a0c67..3ab5f47f19 100644
--- a/libraries/chain/include/graphene/chain/config.hpp
+++ b/libraries/chain/include/graphene/chain/config.hpp
@@ -121,7 +121,7 @@
 #define GRAPHENE_RECENTLY_MISSED_COUNT_INCREMENT             4
 #define GRAPHENE_RECENTLY_MISSED_COUNT_DECREMENT             3
 
-#define GRAPHENE_CURRENT_DB_VERSION                          "BTS2.18"
+#define GRAPHENE_CURRENT_DB_VERSION                          "BTS2.19"
 
 #define GRAPHENE_IRREVERSIBLE_THRESHOLD                      (70 * GRAPHENE_1_PERCENT)
 

From 115ed51b787ce96c2743c988897cfd2f1a6d75df Mon Sep 17 00:00:00 2001
From: Alfredo <oxarbitrage@gmail.com>
Date: Fri, 14 Dec 2018 11:49:10 -0300
Subject: [PATCH 32/38] clarify proxy vote test times

---
 tests/tests/voting_tests.cpp | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/tests/tests/voting_tests.cpp b/tests/tests/voting_tests.cpp
index b4af210637..141c19e0f3 100644
--- a/tests/tests/voting_tests.cpp
+++ b/tests/tests/voting_tests.cpp
@@ -460,7 +460,7 @@ BOOST_AUTO_TEST_CASE(last_voting_date_proxy)
       // witness to vote for
       auto witness1 = witness_id_type(1)(db);
 
-      // alice changes proxy, this is voting activity
+      // round1: alice changes proxy, this is voting activity
       {
          graphene::chain::account_update_operation op;
          op.account = alice_id;
@@ -472,12 +472,12 @@ BOOST_AUTO_TEST_CASE(last_voting_date_proxy)
       }
       // alice last_vote_time is updated
       auto alice_stats_obj = db.get_account_stats_by_owner(alice_id);
-      auto now = db.head_block_time().sec_since_epoch();
-      BOOST_CHECK_EQUAL(alice_stats_obj.last_vote_time.sec_since_epoch(), now);
+      auto round1 = db.head_block_time().sec_since_epoch();
+      BOOST_CHECK_EQUAL(alice_stats_obj.last_vote_time.sec_since_epoch(), round1);
 
       generate_block();
 
-      // alice update account but no proxy or voting changes are done
+      // round 2: alice update account but no proxy or voting changes are done
       {
          graphene::chain::account_update_operation op;
          op.account = alice_id;
@@ -488,13 +488,13 @@ BOOST_AUTO_TEST_CASE(last_voting_date_proxy)
          PUSH_TX( db, trx, ~0 );
       }
       // last_vote_time is not updated
-      now = db.head_block_time().sec_since_epoch();
+      auto round2 = db.head_block_time().sec_since_epoch();
       alice_stats_obj = db.get_account_stats_by_owner(alice_id);
-      BOOST_CHECK(alice_stats_obj.last_vote_time.sec_since_epoch() != now);
+      BOOST_CHECK_EQUAL(alice_stats_obj.last_vote_time.sec_since_epoch(), round1);
 
       generate_block();
 
-      // bob votes
+      // round 3: bob votes
       {
          graphene::chain::account_update_operation op;
          op.account = bob_id;
@@ -507,13 +507,13 @@ BOOST_AUTO_TEST_CASE(last_voting_date_proxy)
       }
 
       // last_vote_time for bob is updated as he voted
-      now = db.head_block_time().sec_since_epoch();
+      auto round3 = db.head_block_time().sec_since_epoch();
       auto bob_stats_obj = db.get_account_stats_by_owner(bob_id);
-      BOOST_CHECK_EQUAL(bob_stats_obj.last_vote_time.sec_since_epoch(), now);
+      BOOST_CHECK_EQUAL(bob_stats_obj.last_vote_time.sec_since_epoch(), round3);
 
       generate_block();
 
-      // proxy votes
+      // round 4: proxy votes
       {
          graphene::chain::account_update_operation op;
          op.account = proxy_id;
@@ -525,17 +525,17 @@ BOOST_AUTO_TEST_CASE(last_voting_date_proxy)
       }
 
       // proxy just voted so the last_vote_time is updated
-      now = db.head_block_time().sec_since_epoch();
+      auto round4 = db.head_block_time().sec_since_epoch();
       auto proxy_stats_obj = db.get_account_stats_by_owner(proxy_id);
-      BOOST_CHECK_EQUAL(proxy_stats_obj.last_vote_time.sec_since_epoch(), now);
+      BOOST_CHECK_EQUAL(proxy_stats_obj.last_vote_time.sec_since_epoch(), round4);
 
       // alice haves proxy, proxy votes but last_vote_time is not updated for alice
       alice_stats_obj = db.get_account_stats_by_owner(alice_id);
-      BOOST_CHECK(alice_stats_obj.last_vote_time.sec_since_epoch() != now);
+      BOOST_CHECK_EQUAL(alice_stats_obj.last_vote_time.sec_since_epoch(), round1);
 
       // bob haves nothing to do with proxy so last_vote_time is not updated
       bob_stats_obj = db.get_account_stats_by_owner(bob_id);
-      BOOST_CHECK(bob_stats_obj.last_vote_time.sec_since_epoch() != now);
+      BOOST_CHECK_EQUAL(bob_stats_obj.last_vote_time.sec_since_epoch(), round3);
 
    } FC_LOG_AND_RETHROW()
 }

From 4a3f45b1766067d1a1f356474166e04df329d624 Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Fri, 14 Dec 2018 18:15:14 +0100
Subject: [PATCH 33/38] Fixed code smell

---
 libraries/db/include/graphene/db/index.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/db/include/graphene/db/index.hpp b/libraries/db/include/graphene/db/index.hpp
index ddee3833ed..babfbda91f 100644
--- a/libraries/db/include/graphene/db/index.hpp
+++ b/libraries/db/include/graphene/db/index.hpp
@@ -204,7 +204,7 @@ namespace graphene { namespace db {
    {
       static_assert( chunkbits < 64, "Do you really want arrays with more than 2^63 elements???" );
 
-      private:
+      // private
          static const size_t MAX_HOLE = 100;
          static const size_t _mask = ((1 << chunkbits) - 1);
          size_t next = 0;

From dd6c7fc4e7bf09b63b67c7cad095faaf9a701ba6 Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Sat, 15 Dec 2018 21:24:45 +0100
Subject: [PATCH 34/38] Properly initialize inner vectors, support nested
 modifications

---
 libraries/db/include/graphene/db/index.hpp | 14 +++++++++-----
 tests/tests/database_tests.cpp             | 10 ++++++++++
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/libraries/db/include/graphene/db/index.hpp b/libraries/db/include/graphene/db/index.hpp
index babfbda91f..3687b5d803 100644
--- a/libraries/db/include/graphene/db/index.hpp
+++ b/libraries/db/include/graphene/db/index.hpp
@@ -23,11 +23,14 @@
  */
 #pragma once
 #include <graphene/db/object.hpp>
+
 #include <fc/interprocess/file_mapping.hpp>
 #include <fc/io/raw.hpp>
 #include <fc/io/json.hpp>
 #include <fc/crypto/sha256.hpp>
+
 #include <fstream>
+#include <stack>
 
 namespace graphene { namespace db {
    class object_database;
@@ -209,7 +212,7 @@ namespace graphene { namespace db {
          static const size_t _mask = ((1 << chunkbits) - 1);
          size_t next = 0;
          vector< vector< const Object* > > content;
-         object_id_type id_being_modified;
+         std::stack< object_id_type > ids_being_modified;
 
       public:
          direct_index() {
@@ -226,7 +229,7 @@ namespace graphene { namespace db {
                if( !(next & _mask) )
                {
                   content.resize((next >> chunkbits) + 1);
-                  content[next >> chunkbits].reserve( 1 << chunkbits );
+                  content[next >> chunkbits].resize( 1 << chunkbits, nullptr );
                }
                next++;
             }
@@ -238,7 +241,7 @@ namespace graphene { namespace db {
                if( !(next & _mask) || (next & (~_mask)) != (instance & (~_mask)) )
                {
                   content.resize((instance >> chunkbits) + 1);
-                  content[instance >> chunkbits].reserve( 1 << chunkbits );
+                  content[instance >> chunkbits].resize( 1 << chunkbits, nullptr );
                }
                while( next <= instance )
                {
@@ -261,12 +264,13 @@ namespace graphene { namespace db {
 
          virtual void about_to_modify( const object& before )
          {
-            id_being_modified = before.id;
+            ids_being_modified.emplace( before.id );
          }
 
          virtual void object_modified( const object& after  )
          {
-            FC_ASSERT( id_being_modified == after.id, "Modification of ID is not supported!");
+            FC_ASSERT( ids_being_modified.top() == after.id, "Modification of ID is not supported!");
+            ids_being_modified.pop();
          }
 
          template< typename object_id >
diff --git a/tests/tests/database_tests.cpp b/tests/tests/database_tests.cpp
index e8c1d24fe2..6158c2eb05 100644
--- a/tests/tests/database_tests.cpp
+++ b/tests/tests/database_tests.cpp
@@ -182,6 +182,16 @@ BOOST_AUTO_TEST_CASE( direct_index_test )
    test_account.name = "account50";
    my_accounts.load( fc::raw::pack( test_account ) );
 
+   // can handle nested modification
+   my_accounts.modify( direct.get( account_id_type(0) ), [&direct,&my_accounts] ( object& outer ) {
+      account_object& _outer = dynamic_cast< account_object& >( outer );
+      my_accounts.modify( direct.get( account_id_type(50) ), [] ( object& inner ) {
+         account_object& _inner = dynamic_cast< account_object& >( inner );
+         _inner.referrer = account_id_type(102);
+      });
+      _outer.options.voting_account = GRAPHENE_PROXY_TO_SELF_ACCOUNT;
+   });
+
    // direct.next is still 103, so 204 is not allowed
    test_account.id = account_id_type(204);
    test_account.name = "account204";

From 69c4011cf483819f0fc4cd31c34dc21bbb68c66b Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Sat, 15 Dec 2018 21:26:23 +0100
Subject: [PATCH 35/38] Removed obsolete hardfork code to avoid problems with
 next commit

---
 libraries/chain/db_block.cpp | 8 --------
 tests/tests/block_tests.cpp  | 5 -----
 2 files changed, 13 deletions(-)

diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp
index efc5562a89..629c9b8605 100644
--- a/libraries/chain/db_block.cpp
+++ b/libraries/chain/db_block.cpp
@@ -683,14 +683,6 @@ processed_transaction database::_apply_transaction(const signed_transaction& trx
    }
    ptrx.operation_results = std::move(eval_state.operation_results);
 
-   if( head_block_time() < HARDFORK_CORE_1040_TIME ) // TODO totally remove this code block after hard fork
-   {
-      //Make sure the temp account has no non-zero balances
-      const auto& index = get_index_type<account_balance_index>().indices().get<by_account_asset>();
-      auto range = index.equal_range( boost::make_tuple( GRAPHENE_TEMP_ACCOUNT ) );
-      std::for_each(range.first, range.second, [](const account_balance_object& b) { FC_ASSERT(b.balance == 0); });
-   }
-
    return ptrx;
 } FC_CAPTURE_AND_RETHROW( (trx) ) }
 
diff --git a/tests/tests/block_tests.cpp b/tests/tests/block_tests.cpp
index e71642726b..8af8e8777c 100644
--- a/tests/tests/block_tests.cpp
+++ b/tests/tests/block_tests.cpp
@@ -1830,11 +1830,6 @@ BOOST_FIXTURE_TEST_CASE( temp_account_balance, database_fixture )
 
    top.to = GRAPHENE_COMMITTEE_ACCOUNT;
    trx.operations.push_back( top );
-   sign( trx, alice_private_key );
-   BOOST_CHECK_THROW( PUSH_TX( db, trx ), fc::assert_exception );
-
-   generate_blocks( HARDFORK_CORE_1040_TIME );
-
    set_expiration( db, trx );
    trx.clear_signatures();
    sign( trx, alice_private_key );

From 2409104bb075a9a49e261185fb3858243fb7b237 Mon Sep 17 00:00:00 2001
From: Peter Conrad <conrad@quisquis.de>
Date: Sat, 15 Dec 2018 21:28:05 +0100
Subject: [PATCH 36/38] Added direct_index-like replacement for
 by_account_asset subindex

---
 libraries/app/database_api.cpp                | 16 +++---
 libraries/chain/account_object.cpp            | 50 +++++++++++++++++++
 libraries/chain/db_balance.cpp                | 19 +++----
 libraries/chain/db_init.cpp                   |  5 +-
 libraries/chain/db_maint.cpp                  | 12 ++---
 .../include/graphene/chain/account_object.hpp | 32 +++++++++---
 6 files changed, 98 insertions(+), 36 deletions(-)

diff --git a/libraries/app/database_api.cpp b/libraries/app/database_api.cpp
index 9316cfa4b6..9a86afba4a 100644
--- a/libraries/app/database_api.cpp
+++ b/libraries/app/database_api.cpp
@@ -799,11 +799,9 @@ std::map<std::string, full_account> database_api_impl::get_full_accounts( const
 
 
       // Add the account's balances
-      auto balance_range = _db.get_index_type<account_balance_index>().indices().get<by_account_asset>().equal_range(boost::make_tuple(account->id));
-      std::for_each(balance_range.first, balance_range.second,
-                    [&acnt](const account_balance_object& balance) {
-                       acnt.balances.emplace_back(balance);
-                    });
+      const auto& balances = _db.get_index_type< primary_index< account_balance_index > >().get_secondary_index< balances_by_account_index >().get_account_balances( account->id );
+      for( const auto balance : balances )
+         acnt.balances.emplace_back( *balance.second );
 
       // Add the account's vesting balances
       auto vesting_range = _db.get_index_type<vesting_balance_index>().indices().get<by_account>().equal_range(account->id);
@@ -955,10 +953,10 @@ vector<asset> database_api_impl::get_account_balances(const std::string& account
    if (assets.empty())
    {
       // if the caller passes in an empty list of assets, return balances for all assets the account owns
-      const account_balance_index& balance_index = _db.get_index_type<account_balance_index>();
-      auto range = balance_index.indices().get<by_account_asset>().equal_range(boost::make_tuple(acnt));
-      for (const account_balance_object& balance : boost::make_iterator_range(range.first, range.second))
-         result.push_back(asset(balance.get_balance()));
+      const auto& balance_index = _db.get_index_type< primary_index< account_balance_index > >();
+      const auto& balances = balance_index.get_secondary_index< balances_by_account_index >().get_account_balances( acnt );
+      for( const auto balance : balances )
+         result.push_back( balance.second->get_balance() );
    }
    else
    {
diff --git a/libraries/chain/account_object.cpp b/libraries/chain/account_object.cpp
index 952a4c7c2c..466f7a6fc9 100644
--- a/libraries/chain/account_object.cpp
+++ b/libraries/chain/account_object.cpp
@@ -269,4 +269,54 @@ void account_referrer_index::object_modified( const object& after  )
 {
 }
 
+const uint8_t  balances_by_account_index::bits = 20;
+const uint64_t balances_by_account_index::mask = (1ULL << balances_by_account_index::bits) - 1;
+
+void balances_by_account_index::object_inserted( const object& obj )
+{
+   const auto& abo = dynamic_cast< const account_balance_object& >( obj );
+   while( balances.size() < (abo.owner.instance.value >> bits) + 1 )
+   {
+      balances.reserve( (abo.owner.instance.value >> bits) + 1 );
+      balances.resize( balances.size() + 1 );
+      balances.back().resize( 1ULL << bits );
+   }
+   balances[abo.owner.instance.value >> bits][abo.owner.instance.value & mask][abo.asset_type] = &abo;
+}
+
+void balances_by_account_index::object_removed( const object& obj )
+{
+   const auto& abo = dynamic_cast< const account_balance_object& >( obj );
+   if( balances.size() < (abo.owner.instance.value >> bits) + 1 ) return;
+   balances[abo.owner.instance.value >> bits][abo.owner.instance.value & mask].erase( abo.asset_type );
+}
+
+void balances_by_account_index::about_to_modify( const object& before )
+{
+   ids_being_modified.emplace( before.id );
+}
+
+void balances_by_account_index::object_modified( const object& after  )
+{
+   FC_ASSERT( ids_being_modified.top() == after.id, "Modification of ID is not supported!");
+   ids_being_modified.pop();
+}
+
+const map< asset_id_type, const account_balance_object* >& balances_by_account_index::get_account_balances( const account_id_type& acct )const
+{
+   static const map< asset_id_type, const account_balance_object* > _empty;
+
+   if( balances.size() < (acct.instance.value >> bits) + 1 ) return _empty;
+   return balances[acct.instance.value >> bits][acct.instance.value & mask];
+}
+
+const account_balance_object* balances_by_account_index::get_account_balance( const account_id_type& acct, const asset_id_type& asset )const
+{
+   if( balances.size() < (acct.instance.value >> bits) + 1 ) return nullptr;
+   const auto& mine = balances[acct.instance.value >> bits][acct.instance.value & mask];
+   const auto itr = mine.find( asset );
+   if( mine.end() == itr ) return nullptr;
+   return itr->second;
+}
+
 } } // graphene::chain
diff --git a/libraries/chain/db_balance.cpp b/libraries/chain/db_balance.cpp
index cf58cb432d..caa1eff63b 100644
--- a/libraries/chain/db_balance.cpp
+++ b/libraries/chain/db_balance.cpp
@@ -33,11 +33,11 @@ namespace graphene { namespace chain {
 
 asset database::get_balance(account_id_type owner, asset_id_type asset_id) const
 {
-   auto& index = get_index_type<account_balance_index>().indices().get<by_account_asset>();
-   auto itr = index.find(boost::make_tuple(owner, asset_id));
-   if( itr == index.end() )
+   auto& index = get_index_type< primary_index< account_balance_index > >().get_secondary_index<balances_by_account_index>();
+   auto abo = index.get_account_balance( owner, asset_id );
+   if( !abo )
       return asset(0, asset_id);
-   return itr->get_balance();
+   return abo->get_balance();
 }
 
 asset database::get_balance(const account_object& owner, const asset_object& asset_obj) const
@@ -55,9 +55,9 @@ void database::adjust_balance(account_id_type account, asset delta )
    if( delta.amount == 0 )
       return;
 
-   auto& index = get_index_type<account_balance_index>().indices().get<by_account_asset>();
-   auto itr = index.find(boost::make_tuple(account, delta.asset_id));
-   if(itr == index.end())
+   auto& index = get_index_type< primary_index< account_balance_index > >().get_secondary_index<balances_by_account_index>();
+   auto abo = index.get_account_balance( account, delta.asset_id );
+   if( !abo )
    {
       FC_ASSERT( delta.amount > 0, "Insufficient Balance: ${a}'s balance of ${b} is less than required ${r}", 
                  ("a",account(*this).name)
@@ -72,8 +72,9 @@ void database::adjust_balance(account_id_type account, asset delta )
       });
    } else {
       if( delta.amount < 0 )
-         FC_ASSERT( itr->get_balance() >= -delta, "Insufficient Balance: ${a}'s balance of ${b} is less than required ${r}", ("a",account(*this).name)("b",to_pretty_string(itr->get_balance()))("r",to_pretty_string(-delta)));
-      modify(*itr, [delta](account_balance_object& b) {
+         FC_ASSERT( abo->get_balance() >= -delta, "Insufficient Balance: ${a}'s balance of ${b} is less than required ${r}",
+                    ("a",account(*this).name)("b",to_pretty_string(abo->get_balance()))("r",to_pretty_string(-delta)));
+      modify(*abo, [delta](account_balance_object& b) {
          b.adjust_balance(delta);
       });
    }
diff --git a/libraries/chain/db_init.cpp b/libraries/chain/db_init.cpp
index c87d9ec6a6..4ecd578fba 100644
--- a/libraries/chain/db_init.cpp
+++ b/libraries/chain/db_init.cpp
@@ -205,7 +205,10 @@ void database::initialize_indexes()
 
    //Implementation object indexes
    add_index< primary_index<transaction_index                             > >();
-   add_index< primary_index<account_balance_index                         > >();
+
+   auto bal_idx = add_index< primary_index<account_balance_index          > >();
+   bal_idx->add_secondary_index<balances_by_account_index>();
+
    add_index< primary_index<asset_bitasset_data_index,                 13 > >(); // 8192
    add_index< primary_index<simple_index<global_property_object          >> >();
    add_index< primary_index<simple_index<dynamic_global_property_object  >> >();
diff --git a/libraries/chain/db_maint.cpp b/libraries/chain/db_maint.cpp
index d011740a6c..4ee7cb2825 100644
--- a/libraries/chain/db_maint.cpp
+++ b/libraries/chain/db_maint.cpp
@@ -693,7 +693,7 @@ void distribute_fba_balances( database& db )
 void create_buyback_orders( database& db )
 {
    const auto& bbo_idx = db.get_index_type< buyback_index >().indices().get<by_id>();
-   const auto& bal_idx = db.get_index_type< account_balance_index >().indices().get< by_account_asset >();
+   const auto& bal_idx = db.get_index_type< primary_index< account_balance_index > >().get_secondary_index< balances_by_account_index >();
 
    for( const buyback_object& bbo : bbo_idx )
    {
@@ -701,7 +701,6 @@ void create_buyback_orders( database& db )
       assert( asset_to_buy.buyback_account.valid() );
 
       const account_object& buyback_account = (*(asset_to_buy.buyback_account))(db);
-      asset_id_type next_asset = asset_id_type();
 
       if( !buyback_account.allowed_assets.valid() )
       {
@@ -709,16 +708,11 @@ void create_buyback_orders( database& db )
          continue;
       }
 
-      while( true )
+      for( const auto& entry : bal_idx.get_account_balances( buyback_account.id ) )
       {
-         auto it = bal_idx.lower_bound( boost::make_tuple( buyback_account.id, next_asset ) );
-         if( it == bal_idx.end() )
-            break;
-         if( it->owner != buyback_account.id )
-            break;
+         const auto* it = entry.second;
          asset_id_type asset_to_sell = it->asset_type;
          share_type amount_to_sell = it->balance;
-         next_asset = asset_to_sell + 1;
          if( asset_to_sell == asset_to_buy.id )
             continue;
          if( amount_to_sell == 0 )
diff --git a/libraries/chain/include/graphene/chain/account_object.hpp b/libraries/chain/include/graphene/chain/account_object.hpp
index cae9d35984..1c4526fde0 100644
--- a/libraries/chain/include/graphene/chain/account_object.hpp
+++ b/libraries/chain/include/graphene/chain/account_object.hpp
@@ -341,7 +341,30 @@ namespace graphene { namespace chain {
          map< account_id_type, set<account_id_type> > referred_by;
    };
 
-   struct by_account_asset;
+   /**
+    *  @brief This secondary index will allow fast access to the balance objects
+    *         that belonging to an account.
+    */
+   class balances_by_account_index : public secondary_index
+   {
+      public:
+         virtual void object_inserted( const object& obj ) override;
+         virtual void object_removed( const object& obj ) override;
+         virtual void about_to_modify( const object& before ) override;
+         virtual void object_modified( const object& after  ) override;
+
+         const map< asset_id_type, const account_balance_object* >& get_account_balances( const account_id_type& acct )const;
+         const account_balance_object* get_account_balance( const account_id_type& acct, const asset_id_type& asset )const;
+
+      private:
+         static const uint8_t  bits;
+         static const uint64_t mask;
+
+         /** Maps each account to its balance objects */
+         vector< vector< map< asset_id_type, const account_balance_object* > > > balances;
+         std::stack< object_id_type > ids_being_modified;
+   };
+
    struct by_asset_balance;
    struct by_maintenance_flag;
    /**
@@ -353,13 +376,6 @@ namespace graphene { namespace chain {
          ordered_unique< tag<by_id>, member< object, object_id_type, &object::id > >,
          ordered_non_unique< tag<by_maintenance_flag>,
                              member< account_balance_object, bool, &account_balance_object::maintenance_flag > >,
-         ordered_unique< tag<by_account_asset>,
-            composite_key<
-               account_balance_object,
-               member<account_balance_object, account_id_type, &account_balance_object::owner>,
-               member<account_balance_object, asset_id_type, &account_balance_object::asset_type>
-            >
-         >,
          ordered_unique< tag<by_asset_balance>,
             composite_key<
                account_balance_object,

From fe4c4e9eefa858ec7c4e784a05ee5b686d50a44c Mon Sep 17 00:00:00 2001
From: Alfredo Garcia <oxarbitrage@gmail.com>
Date: Mon, 17 Dec 2018 14:27:33 -0300
Subject: [PATCH 37/38] add link to hackthedex.io to README

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index a516790596..e5147f0112 100644
--- a/README.md
+++ b/README.md
@@ -22,6 +22,8 @@ Visit [BitShares.org](https://bitshares.org/) to learn about BitShares and join
 
 Information for developers can be found in the [Bitshares Developer Portal](https://dev.bitshares.works/). Users interested in how bitshares works can go to the [BitShares Documentation](https://how.bitshares.works/) site.
 
+For security issues and bug bounty program please visit [Hack the DEX](https://hackthedex.io).
+
 Getting Started
 ---------------
 Build instructions and additional documentation are available in the

From 1e8ccab7b411de624b418be2ad4cbd68d0cf442f Mon Sep 17 00:00:00 2001
From: Alfredo <oxarbitrage@gmail.com>
Date: Tue, 18 Dec 2018 15:22:56 -0300
Subject: [PATCH 38/38] change adaptor response to be mutable_variant_object

---
 libraries/plugins/es_objects/es_objects.cpp              | 9 +++------
 .../include/graphene/es_objects/es_objects.hpp           | 8 +++-----
 2 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp
index 01ec4722a6..5695064325 100644
--- a/libraries/plugins/es_objects/es_objects.cpp
+++ b/libraries/plugins/es_objects/es_objects.cpp
@@ -198,14 +198,11 @@ void es_objects_plugin_impl::prepareTemplate(T blockchain_object, string index_n
       bulk_header["_id"] = string(blockchain_object.id);
    }
 
-   auto blockchain_object_string = fc::json::to_string(blockchain_object, fc::json::legacy_generator);
-   variant blockchain_object_variant = fc::json::from_string(blockchain_object_string);
-   fc::mutable_variant_object o;
-
    adaptor_struct adaptor;
-   auto adapted = adaptor.adapt(blockchain_object_variant.get_object());
+   fc::variant blockchain_object_variant;
+   fc::to_variant( blockchain_object, blockchain_object_variant, GRAPHENE_NET_MAX_NESTED_OBJECTS );
+   fc::mutable_variant_object o = adaptor.adapt(blockchain_object_variant.get_object());
 
-   fc::from_variant(adapted, o, FC_PACK_MAX_DEPTH);
    o["object_id"] = string(blockchain_object.id);
    o["block_time"] = block_time;
    o["block_number"] = block_number;
diff --git a/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp b/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp
index a66e299551..fa91e3bde4 100644
--- a/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp
+++ b/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp
@@ -54,7 +54,7 @@ class es_objects_plugin : public graphene::app::plugin
 };
 
 struct adaptor_struct {
-   variant adapt(const variant_object &obj) {
+    fc::mutable_variant_object adapt(const variant_object &obj) {
       fc::mutable_variant_object o(obj);
       vector<string> keys_to_rename;
       for (auto i = o.begin(); i != o.end(); ++i) {
@@ -94,10 +94,8 @@ struct adaptor_struct {
       {
          o["operations"] = fc::json::to_string(o["operations"]);
       }
-
-      variant v;
-      fc::to_variant(o, v, FC_PACK_MAX_DEPTH);
-      return v;
+      
+      return o;
    }
 
    void adapt(fc::variants &v) {