Merge pull request #342 from eosnetworkfoundation/blog_trimming

block log pruning: optionally reduce disk usage of block log file during operation by periodically trimming blocks

libraries/chain/controller.cpp

@@ -299,7 +299,7 @@ struct controller_impl {
     db( cfg.state_dir,
         cfg.read_only ? database::read_only : database::read_write,
         cfg.state_size, false, cfg.db_map_mode ),
-    blog( cfg.blocks_dir ),
+    blog( cfg.blocks_dir, cfg.prune_config ),
     fork_db( cfg.blocks_dir / config::reversible_blocks_dir_name ),
     wasmif( cfg.wasm_runtime, cfg.eosvmoc_tierup, db, cfg.state_dir, cfg.eosvmoc_config, !cfg.profile_accounts.empty() ),
     resource_limits( db, [&s]() { return s.get_deep_mind_logger(); }),

libraries/chain/include/eosio/chain/block_log.hpp

@@ -30,15 +30,25 @@ namespace eosio { namespace chain {
     *
     * The main file is the only file that needs to persist. The index file can be reconstructed during a
     * linear scan of the main file.
+    *
+    * An optional "pruned" mode can be activated which stores a 4 byte trailer on the log file indicating
+    * how many blocks at the end of the log are valid. Any earlier blocks in the log are assumed destroyed
+    * and unreadable due to reclamation for purposes of saving space.
     */
 
+   struct block_log_prune_config {
+      uint32_t                prune_blocks;                  //number of blocks to prune to when doing a prune
+      size_t                  prune_threshold = 4*1024*1024; //(approximately) how many bytes need to be added before a prune is performed
+      std::optional<size_t>   vacuum_on_close;               //when set, a vacuum is performed on dtor if log contains less than this many live bytes
+   };
+
    class block_log {
       public:
-         block_log(const fc::path& data_dir);
+         block_log(const fc::path& data_dir, std::optional<block_log_prune_config> prune_config);
          block_log(block_log&& other);
          ~block_log();
 
-         uint64_t append(const signed_block_ptr& b);
+         void append(const signed_block_ptr& b);
          void flush();
          void reset( const genesis_state& gs, const signed_block_ptr& genesis_block );
          void reset( const chain_id_type& chain_id, uint32_t first_block_num );
@@ -79,6 +89,8 @@ namespace eosio { namespace chain {
 
          static bool is_supported_version(uint32_t version);
 
+         static bool is_pruned_log(const fc::path& data_dir);
+
          static bool trim_blocklog_front(const fc::path& block_dir, const fc::path& temp_dir, uint32_t truncate_at_block);
 
    private:

libraries/chain/include/eosio/chain/controller.hpp

@@ -1,5 +1,6 @@
 #pragma once
 #include <eosio/chain/block_state.hpp>
+#include <eosio/chain/block_log.hpp>
 #include <eosio/chain/trace.hpp>
 #include <eosio/chain/genesis_state.hpp>
 #include <chainbase/pinnable_mapped_file.hpp>
@@ -69,6 +70,7 @@ namespace eosio { namespace chain {
             flat_set< pair<account_name, action_name> > action_blacklist;
             flat_set<public_key_type> key_blacklist;
             path                     blocks_dir             =  chain::config::default_blocks_dir_name;
+            std::optional<block_log_prune_config>  prune_config;
             path                     state_dir              =  chain::config::default_state_dir_name;
             uint64_t                 state_size             =  chain::config::default_state_size;
             uint64_t                 state_guard_size       =  chain::config::default_state_guard_size;

plugins/chain_plugin/chain_plugin.cpp

@@ -331,8 +331,11 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip
          ("transaction-finality-status-success-duration-sec", bpo::value<uint64_t>()->default_value(config::default_max_transaction_finality_status_success_duration_sec),
           "Duration (in seconds) a successful transaction's Finality Status will remain available from being first identified.")
          ("transaction-finality-status-failure-duration-sec", bpo::value<uint64_t>()->default_value(config::default_max_transaction_finality_status_failure_duration_sec),
-          "Duration (in seconds) a failed transaction's Finality Status will remain available from being first identified.")
-         ;
+          "Duration (in seconds) a failed transaction's Finality Status will remain available from being first identified.");
+
+   if(cfile::supports_hole_punching())
+      cfg.add_options()("block-log-retain-blocks", bpo::value<uint32_t>(), "if set, periodically prune the block log to store only configured number of most recent blocks");
+
 
 // TODO: rate limiting
          /*("per-authorized-account-transaction-msg-rate-limit-time-frame-sec", bpo::value<uint32_t>()->default_value(default_per_auth_account_time_frame_seconds),
@@ -860,6 +863,12 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
       // move fork_db to new location
       upgrade_from_reversible_to_fork_db( my.get() );
 
+      if(options.count( "block-log-retain-blocks" )) {
+         my->chain_config->prune_config.emplace();
+         my->chain_config->prune_config->prune_blocks = options.at( "block-log-retain-blocks" ).as<uint32_t>();
+         EOS_ASSERT(my->chain_config->prune_config->prune_blocks, plugin_config_exception, "block-log-retain-blocks cannot be 0");
+      }
+
       if( options.at( "delete-all-blocks" ).as<bool>()) {
          ilog( "Deleting state database and blocks" );
          if( options.at( "truncate-at-block" ).as<uint32_t>() > 0 )

programs/eosio-blocklog/main.cpp

@@ -37,6 +37,7 @@ struct blocklog {
    void read_log();
    void set_program_options(options_description& cli);
    void initialize(const variables_map& options);
+   void do_vacuum();
 
    bfs::path                        blocks_dir;
    bfs::path                        output_file;
@@ -47,7 +48,10 @@ struct blocklog {
    bool                             make_index = false;
    bool                             trim_log = false;
    bool                             smoke_test = false;
+   bool                             vacuum = false;
    bool                             help = false;
+
+   std::optional<block_log_prune_config> blog_keep_prune_conf;
 };
 
 struct report_time {
@@ -65,9 +69,15 @@ struct report_time {
     const std::string                                    _desc;
 };
 
+void blocklog::do_vacuum() {
+   EOS_ASSERT( blog_keep_prune_conf, block_log_exception, "blocks.log is not a pruned log; nothing to vacuum" );
+   block_log blocks(blocks_dir, std::optional<block_log_prune_config>()); //passing an unset block_log_prune_config turns off pruning this performs a vacuum
+   ilog("Successfully vacuumed block log");
+}
+
 void blocklog::read_log() {
    report_time rt("reading log");
-   block_log block_logger(blocks_dir);
+   block_log block_logger(blocks_dir, blog_keep_prune_conf);
    const auto end = block_logger.read_head();
    EOS_ASSERT( end, block_log_exception, "No blocks found in block log" );
    EOS_ASSERT( end->block_num() > 1, block_log_exception, "Only one block found in block log" );
@@ -186,6 +196,8 @@ void blocklog::set_program_options(options_description& cli)
           "Trim blocks.log and blocks.index. Must give 'blocks-dir' and 'first and/or 'last'.")
          ("smoke-test", bpo::bool_switch(&smoke_test)->default_value(false),
           "Quick test that blocks.log and blocks.index are well formed and agree with each other.")
+         ("vacuum", bpo::bool_switch(&vacuum)->default_value(false),
+          "Vacuum a pruned blocks.log in to an un-pruned blocks.log")
          ("help,h", bpo::bool_switch(&help)->default_value(false), "Print this help message and exit.")
          ;
 }
@@ -205,6 +217,13 @@ void blocklog::initialize(const variables_map& options) {
          else
             output_file = bld;
       }
+
+      //if the log is pruned, keep it that way by passing in a config with a large block pruning value. There is otherwise no
+      // way to tell block_log "keep the current non/pruneness of the log"
+      if(block_log::is_pruned_log(blocks_dir)) {
+         blog_keep_prune_conf.emplace();
+         blog_keep_prune_conf->prune_blocks = UINT32_MAX;
+      }
    } FC_LOG_AND_RETHROW()
 
 }
@@ -302,6 +321,11 @@ int main(int argc, char** argv) {
          }
          return 0;
       }
+      if (blog.vacuum) {
+         blog.initialize(vmap);
+         blog.do_vacuum();
+         return 0;
+      }
       if (blog.make_index) {
          const bfs::path blocks_dir = vmap.at("blocks-dir").as<bfs::path>();
          bfs::path out_file = blocks_dir / "blocks.index";