Resolve 1371: sorting of public_keys is inefficient

libraries/app/application.cpp

@@ -26,6 +26,7 @@
 #include <graphene/app/application.hpp>
 #include <graphene/app/plugin.hpp>
 
+#include <graphene/chain/db_with.hpp>
 #include <graphene/chain/genesis_state.hpp>
 #include <graphene/chain/protocol/fee_schedule.hpp>
 #include <graphene/chain/protocol/types.hpp>
@@ -394,12 +395,34 @@ void application_impl::startup()
       _chain_db->enable_standby_votes_tracking( _options->at("enable-standby-votes-tracking").as<bool>() );
    }
 
-   if( _options->count("replay-blockchain") )
+   if( _options->count("replay-blockchain") || _options->count("revalidate-blockchain") )
       _chain_db->wipe( _data_dir / "blockchain", false );
 
    try
    {
-      _chain_db->open( _data_dir / "blockchain", initial_state, GRAPHENE_CURRENT_DB_VERSION );
+      // these flags are used in open() only, i. e. during replay
+      uint32_t skip;
+      if( _options->count("revalidate-blockchain") ) // see also handle_block()
+      {
+         if( !loaded_checkpoints.empty() )
+            wlog( "Warning - revalidate will not validate before last checkpoint" );
+         if( _options->count("force-validate") )
+            skip = graphene::chain::database::skip_nothing;
+         else
+            skip = graphene::chain::database::skip_transaction_signatures;
+      }
+      else // no revalidate, skip most checks
+         skip = graphene::chain::database::skip_witness_signature |
+                graphene::chain::database::skip_block_size_check |
+                graphene::chain::database::skip_merkle_check |
+                graphene::chain::database::skip_transaction_signatures |
+                graphene::chain::database::skip_transaction_dupe_check |
+                graphene::chain::database::skip_tapos_check |
+                graphene::chain::database::skip_witness_schedule_check;
+
+      graphene::chain::detail::with_skip_flags( *_chain_db, skip, [this,&initial_state] () {
+         _chain_db->open( _data_dir / "blockchain", initial_state, GRAPHENE_CURRENT_DB_VERSION );
+      });
    }
    catch( const fc::exception& e )
    {
@@ -961,9 +984,10 @@ void application::set_program_options(boost::program_options::options_descriptio
           "Path to create a Genesis State at. If a well-formed JSON file exists at the path, it will be parsed and any "
           "missing fields in a Genesis State will be added, and any unknown fields will be removed. If no file or an "
           "invalid file is found, it will be replaced with an example Genesis State.")
-         ("replay-blockchain", "Rebuild object graph by replaying all blocks")
+         ("replay-blockchain", "Rebuild object graph by replaying all blocks without validation")
+         ("revalidate-blockchain", "Rebuild object graph by replaying all blocks with full validation")
          ("resync-blockchain", "Delete all blocks and re-sync with network from scratch")
-         ("force-validate", "Force validation of all transactions")
+         ("force-validate", "Force validation of all transactions during normal operation")
          ("genesis-timestamp", bpo::value<uint32_t>(),
           "Replace timestamp from genesis.json with current time plus this many seconds (experts only!)")
          ;

libraries/app/database_api.cpp

@@ -1994,18 +1994,18 @@ set<public_key_type> database_api_impl::get_potential_signatures( const signed_t
    trx.get_required_signatures(
       _db.get_chain_id(),
       flat_set<public_key_type>(),
-      [&]( account_id_type id )
+      [this,&result]( account_id_type id )
       {
          const auto& auth = id(_db).active;
-         for( const auto& k : auth.get_keys() )
-            result.insert(k);
+         for( const auto& pair : auth.key_auths )
+            result.insert( pair.first );
          return &auth;
       },
-      [&]( account_id_type id )
+      [this,&result]( account_id_type id )
       {
          const auto& auth = id(_db).owner;
-         for( const auto& k : auth.get_keys() )
-            result.insert(k);
+         for( const auto& pair : auth.key_auths )
+            result.insert( pair.first );
          return &auth;
       },
       _db.get_global_properties().parameters.max_authority_depth
@@ -2014,11 +2014,11 @@ set<public_key_type> database_api_impl::get_potential_signatures( const signed_t
    // Insert keys in required "other" authories
    flat_set<account_id_type> required_active;
    flat_set<account_id_type> required_owner;
-   vector<authority> other;
+   vector<const authority*> other;
    trx.get_required_authorities( required_active, required_owner, other );
-   for( const auto& auth : other )
-      for( const auto& key : auth.get_keys() )
-         result.insert( key );
+   for( const auto auth : other )
+      for( const auto& pair : auth->key_auths )
+         result.insert( pair.first );
 
    return result;
 }
@@ -2029,18 +2029,18 @@ set<address> database_api_impl::get_potential_address_signatures( const signed_t
    trx.get_required_signatures(
       _db.get_chain_id(),
       flat_set<public_key_type>(),
-      [&]( account_id_type id )
+      [this,&result]( account_id_type id )
       {
          const auto& auth = id(_db).active;
-         for( const auto& k : auth.get_addresses() )
-            result.insert(k);
+         for( const auto& pair : auth.key_auths )
+            result.insert( pair.first );
          return &auth;
       },
-      [&]( account_id_type id )
+      [this,&result]( account_id_type id )
       {
          const auto& auth = id(_db).owner;
-         for( const auto& k : auth.get_addresses() )
-            result.insert(k);
+         for( const auto& pair : auth.key_auths )
+            result.insert( pair.first );
          return &auth;
       },
       _db.get_global_properties().parameters.max_authority_depth

libraries/chain/CMakeLists.txt

@@ -42,6 +42,7 @@ add_library( graphene_chain
              protocol/asset.cpp
              protocol/assert.cpp
              protocol/account.cpp
+             protocol/balance.cpp
              protocol/transfer.cpp
              protocol/committee_member.cpp
              protocol/witness.cpp

libraries/chain/account_object.cpp

@@ -121,9 +121,9 @@ set<account_id_type> account_member_index::get_account_members(const account_obj
       result.insert(auth.first);
    return result;
 }
-set<public_key_type, account_member_index::key_compare> account_member_index::get_key_members(const account_object& a)const
+set<public_key_type> account_member_index::get_key_members(const account_object& a)const
 {
-   set<public_key_type, key_compare> result;
+   set<public_key_type> result;
    for( auto auth : a.owner.key_auths )
       result.insert(auth.first);
    for( auto auth : a.active.key_auths )
@@ -215,7 +215,7 @@ void account_member_index::object_modified(const object& after)
 
 
     {
-       set<public_key_type, key_compare> after_key_members = get_key_members(a);
+       set<public_key_type> after_key_members = get_key_members(a);
 
        vector<public_key_type> removed; removed.reserve(before_key_members.size());
        std::set_difference(before_key_members.begin(), before_key_members.end(),

libraries/chain/db_block.cpp

@@ -465,15 +465,17 @@ signed_block database::_generate_block(
 void database::pop_block()
 { try {
    _pending_tx_session.reset();
-   auto head_id = head_block_id();
-   optional<signed_block> head_block = fetch_block_by_id( head_id );
-   GRAPHENE_ASSERT( head_block.valid(), pop_empty_chain, "there are no blocks to pop" );
-
-   _fork_db.pop_block();
+   auto fork_db_head = _fork_db.head();
+   FC_ASSERT( fork_db_head, "Trying to pop() from empty fork database!?" );
+   if( fork_db_head->id == head_block_id() )
+      _fork_db.pop_block();
+   else
+   {
+      fork_db_head = _fork_db.fetch_block( head_block_id() );
+      FC_ASSERT( fork_db_head, "Trying to pop() block that's not in fork database!?" );
+   }
    pop_undo();
-
-   _popped_tx.insert( _popped_tx.begin(), head_block->transactions.begin(), head_block->transactions.end() );
-
+   _popped_tx.insert( _popped_tx.begin(), fork_db_head->data.transactions.begin(), fork_db_head->data.transactions.end() );
 } FC_CAPTURE_AND_RETHROW() }
 
 void database::clear_pending()
@@ -621,8 +623,7 @@ processed_transaction database::_apply_transaction(const signed_transaction& trx
 { try {
    uint32_t skip = get_node_properties().skip_flags;
 
-   if( true || !(skip&skip_validate) )   /* issue #505 explains why this skip_flag is disabled */
-      trx.validate();
+   trx.validate();
 
    auto& trx_idx = get_mutable_index_type<transaction_index>();
    const chain_id_type& chain_id = get_chain_id();
@@ -636,7 +637,7 @@ processed_transaction database::_apply_transaction(const signed_transaction& trx
    const chain_parameters& chain_parameters = get_global_properties().parameters;
    eval_state._trx = &trx;
 
-   if( !(skip & (skip_transaction_signatures | skip_authority_check) ) )
+   if( !(skip & skip_transaction_signatures) )
    {
       auto get_active = [&]( account_id_type id ) { return &id(*this).active; };
       auto get_owner  = [&]( account_id_type id ) { return &id(*this).owner;  };

libraries/chain/db_init.cpp

@@ -233,7 +233,7 @@ void database::init_genesis(const genesis_state_type& genesis_state)
    _undo_db.disable();
    struct auth_inhibitor {
       auth_inhibitor(database& db) : db(db), old_flags(db.node_properties().skip_flags)
-      { db.node_properties().skip_flags |= skip_authority_check; }
+      { db.node_properties().skip_flags |= skip_transaction_signatures; }
       ~auth_inhibitor()
       { db.node_properties().skip_flags = old_flags; }
    private:
@@ -500,7 +500,7 @@ void database::init_genesis(const genesis_state_type& genesis_state)
             cop.name = asset.symbol + "-collateral-holder-" + std::to_string(collateral_holder_number);
             boost::algorithm::to_lower(cop.name);
             cop.registrar = GRAPHENE_TEMP_ACCOUNT;
-            cop.owner = authority(1, collateral_rec.owner, 1);
+            cop.owner = authority( collateral_rec.owner );
             cop.active = cop.owner;
             account_id_type owner_account_id = apply_operation(genesis_eval_state, cop).get<object_id_type>();
 

libraries/chain/db_management.cpp

@@ -74,14 +74,7 @@ void database::reindex( fc::path data_dir )
    else
       _undo_db.disable();
 
-   uint32_t skip = skip_witness_signature |
-                   skip_block_size_check |
-                   skip_merkle_check |
-                   skip_transaction_signatures |
-                   skip_transaction_dupe_check |
-                   skip_tapos_check |
-                   skip_witness_schedule_check |
-                   skip_authority_check;
+   uint32_t skip = node_properties().skip_flags;
 
    size_t total_processed_block_size;
    size_t total_block_size = _block_id_to_block.total_block_size();

libraries/chain/db_notify.cpp

@@ -26,6 +26,13 @@ struct get_impacted_account_visitor
    get_impacted_account_visitor( flat_set<account_id_type>& impact ):_impacted(impact) {}
    typedef void result_type;
 
+   static void add_authority_accounts( flat_set<account_id_type>& result,
+                                       const authority& a )
+   {
+      for( auto& item : a.account_auths )
+         result.insert( item.first );
+   }
+
    void operator()( const transfer_operation& op )
    {
       _impacted.insert( op.to );
@@ -150,11 +157,11 @@ struct get_impacted_account_visitor
    void operator()( const proposal_create_operation& op )
    {
       _impacted.insert( op.fee_payer() ); // fee_paying_account
-      vector<authority> other;
+      vector<const authority*> other;
       for( const auto& proposed_op : op.proposed_ops )
          operation_get_required_authorities( proposed_op.op, _impacted, _impacted, other );
       for( auto& o : other )
-         add_authority_accounts( _impacted, o );
+         add_authority_accounts( _impacted, *o );
    }
    void operator()( const proposal_update_operation& op )
    {

libraries/chain/fork_database.cpp

@@ -39,7 +39,7 @@ void fork_database::pop_block()
    FC_ASSERT( _head, "no block to pop" );
    auto prev = _head->prev.lock();
    FC_ASSERT( prev, "popping block would leave head block null" );
-    _head = prev;
+   _head = prev;
 }
 
 void     fork_database::start_block(signed_block b)
@@ -50,7 +50,7 @@ void     fork_database::start_block(signed_block b)
 }
 
 /**
- * Pushes the block into the fork database and caches it if it doesn't link
+ * Pushes the block into the fork database
  *
  */
 shared_ptr<fork_item>  fork_database::push_block(const signed_block& b)
@@ -64,7 +64,6 @@ shared_ptr<fork_item>  fork_database::push_block(const signed_block& b)
       wlog( "Pushing block to fork database that failed to link: ${id}, ${num}", ("id",b.id())("num",b.block_num()) );
       wlog( "Head: ${num}, ${id}", ("num",_head->data.block_num())("id",_head->data.id()) );
       throw;
-      _unlinked_index.insert( item );
    }
    return _head;
 }
@@ -92,77 +91,34 @@ void  fork_database::_push_block(const item_ptr& item)
    {
       _head = item;
       uint32_t min_num = _head->num - std::min( _max_size, _head->num );
-//      ilog( "min block in fork DB ${n}, max_size: ${m}", ("n",min_num)("m",_max_size) );
       auto& num_idx = _index.get<block_num>();
       while( num_idx.size() && (*num_idx.begin())->num < min_num )
          num_idx.erase( num_idx.begin() );
-
-      _unlinked_index.get<block_num>().erase(_head->num - _max_size);
    }
-   //_push_next( item );
-}
-
-/**
- *  Iterate through the unlinked cache and insert anything that
- *  links to the newly inserted item.  This will start a recursive
- *  set of calls performing a depth-first insertion of pending blocks as
- *  _push_next(..) calls _push_block(...) which will in turn call _push_next
- */
-void fork_database::_push_next( const item_ptr& new_item )
-{
-    auto& prev_idx = _unlinked_index.get<by_previous>();
-
-    auto itr = prev_idx.find( new_item->id );
-    while( itr != prev_idx.end() )
-    {
-       auto tmp = *itr;
-       prev_idx.erase( itr );
-       _push_block( tmp );
-
-       itr = prev_idx.find( new_item->id );
-    }
 }
 
 void fork_database::set_max_size( uint32_t s )
 {
    _max_size = s;
    if( !_head ) return;
 
-   { /// index
-      auto& by_num_idx = _index.get<block_num>();
-      auto itr = by_num_idx.begin();
-      while( itr != by_num_idx.end() )
-      {
-         if( (*itr)->num < std::max(int64_t(0),int64_t(_head->num) - _max_size) )
-            by_num_idx.erase(itr);
-         else
-            break;
-         itr = by_num_idx.begin();
-      }
-   }
-   { /// unlinked_index
-      auto& by_num_idx = _unlinked_index.get<block_num>();
-      auto itr = by_num_idx.begin();
-      while( itr != by_num_idx.end() )
-      {
-         if( (*itr)->num < std::max(int64_t(0),int64_t(_head->num) - _max_size) )
-            by_num_idx.erase(itr);
-         else
-            break;
-         itr = by_num_idx.begin();
-      }
+   auto& by_num_idx = _index.get<block_num>();
+   auto itr = by_num_idx.begin();
+   while( itr != by_num_idx.end() )
+   {
+      if( (*itr)->num < std::max(int64_t(0),int64_t(_head->num) - _max_size) )
+         by_num_idx.erase(itr);
+      else
+         break;
+      itr = by_num_idx.begin();
    }
 }
 
 bool fork_database::is_known_block(const block_id_type& id)const
 {
    auto& index = _index.get<block_id>();
    auto itr = index.find(id);
-   if( itr != index.end() )
-      return true;
-   auto& unlinked_index = _unlinked_index.get<block_id>();
-   auto unlinked_itr = unlinked_index.find(id);
-   return unlinked_itr != unlinked_index.end();
+   return itr != index.end();
 }
 
 item_ptr fork_database::fetch_block(const block_id_type& id)const
@@ -171,10 +127,6 @@ item_ptr fork_database::fetch_block(const block_id_type& id)const
    auto itr = index.find(id);
    if( itr != index.end() )
       return *itr;
-   auto& unlinked_index = _unlinked_index.get<block_id>();
-   auto unlinked_itr = unlinked_index.find(id);
-   if( unlinked_itr != unlinked_index.end() )
-      return *unlinked_itr;
    return item_ptr();
 }