Merge branch 'GH-529-optimize-block-processing' into GH-529-optimize-…

…block-processing-part2

libraries/chain/block_log.cpp

@@ -185,6 +185,16 @@ namespace eosio { namespace chain {
          return block;
       }
 
+      template <typename Stream>
+      std::vector<char> read_serialized_block(Stream&& ds, uint64_t block_size) {
+         std::vector<char> buff;
+         buff.resize(block_size);
+
+         ds.read(buff.data(), block_size);
+
+         return buff;
+      }
+
       template <typename Stream>
       signed_block_header read_block_header(Stream&& ds, uint32_t expect_block_num) {
          signed_block_header bh;
@@ -197,6 +207,7 @@ namespace eosio { namespace chain {
          return bh;
       }
 
+
       /// Provide the read only view of the blocks.log file
       class block_log_data : public chain::log_data_base<block_log_data> {
          block_log_preamble preamble;
@@ -245,10 +256,12 @@ namespace eosio { namespace chain {
             //   block_id_type        previous;                   //bytes 14:45, low 4 bytes is big endian block number
             //   of previous block
 
-            EOS_ASSERT(position <= size(), block_log_exception, "Invalid block position ${position}",
-                       ("position", position));
+            int blknum_offset = 14;
+
+            EOS_ASSERT(position + blknum_offset + sizeof(uint32_t) <= size(), block_log_exception,
+                       "Read outside of file: position ${position}, blknum_offset ${o}, file size ${s}",
+                       ("position", position)("o", blknum_offset)("s", size()));
 
-            int      blknum_offset  = 14;
             uint32_t prev_block_num = read_data_at<uint32_t>(file, position + blknum_offset);
             return fc::endian_reverse_u32(prev_block_num) + 1;
          }
@@ -485,6 +498,7 @@ namespace eosio { namespace chain {
          virtual void     flush()                                                             = 0;
 
          virtual signed_block_ptr                   read_block_by_num(uint32_t block_num)        = 0;
+         virtual std::vector<char>                  read_serialized_block_by_num(uint32_t block_num) = 0;
          virtual std::optional<signed_block_header> read_block_header_by_num(uint32_t block_num) = 0;
 
          virtual uint32_t version() const = 0;
@@ -518,6 +532,7 @@ namespace eosio { namespace chain {
          void flush() final {}
 
          signed_block_ptr read_block_by_num(uint32_t block_num) final { return {}; };
+         std::vector<char> read_serialized_block_by_num(uint32_t block_num) final { return {}; };
          std::optional<signed_block_header> read_block_header_by_num(uint32_t block_num) final { return {}; };
 
          uint32_t         version() const final { return 0; }
@@ -562,6 +577,7 @@ namespace eosio { namespace chain {
          virtual uint32_t         working_block_file_first_block_num() { return preamble.first_block_num; }
          virtual void             post_append(uint64_t pos) {}
          virtual signed_block_ptr retry_read_block_by_num(uint32_t block_num) { return {}; }
+         virtual std::vector<char> retry_read_serialized_block_by_num(uint32_t block_num) { return {}; }
          virtual std::optional<signed_block_header> retry_read_block_header_by_num(uint32_t block_num) { return {}; }
 
          void append(const signed_block_ptr& b, const block_id_type& id,
@@ -603,6 +619,46 @@ namespace eosio { namespace chain {
             return pos;
          }
 
+         block_pos_size_t get_block_position_and_size(uint32_t block_num) {
+            uint64_t pos = get_block_pos(block_num);
+
+            if (pos == block_log::npos) {
+               return block_pos_size_t {.position = block_log::npos, .size = 0};
+            }
+
+            assert(head);
+            uint32_t last_block_num = block_header::num_from_id(head->id);
+            EOS_ASSERT(block_num <= last_block_num, block_log_exception,
+                       "block_num ${bn} should not be greater than last_block_num ${lbn}",
+                       ("bn", block_num)("lbn", last_block_num));
+
+            uint64_t block_size = 0;
+            constexpr uint32_t block_pos_size = sizeof(uint64_t); // size of block position field in the block log file
+
+            if (block_num < last_block_num) {
+               // current block is not the last block in the log file.
+               uint64_t next_block_pos = get_block_pos(block_num + 1);
+
+               EOS_ASSERT(next_block_pos > pos + block_pos_size, block_log_exception,
+                          "next block position ${np} should be greater than current block position ${p} plus block position field size ${bps}",
+                          ("np", next_block_pos)("p", pos)("bps", block_pos_size));
+
+               block_size = next_block_pos - pos - block_pos_size;
+            } else {
+               // current block is the last block in the file.
+
+               block_file.seek_end(0);
+               auto file_size = block_file.tellp();
+               EOS_ASSERT(file_size > pos + block_pos_size, block_log_exception,
+                          "block log file size ${fs} should be greater than current block position ${p} plus block position field size ${bps}",
+                          ("fs", file_size)("p", pos)("bps", block_pos_size));
+
+               block_size = file_size - pos - block_pos_size;
+            }
+
+            return block_pos_size_t {.position = pos, .size = block_size};
+         }
+
          signed_block_ptr read_block_by_num(uint32_t block_num) final {
             try {
                uint64_t pos = get_block_pos(block_num);
@@ -615,6 +671,18 @@ namespace eosio { namespace chain {
             FC_LOG_AND_RETHROW()
          }
 
+         std::vector<char> read_serialized_block_by_num(uint32_t block_num) final {
+            try {
+               auto [ position, size ] = get_block_position_and_size(block_num);
+               if (position != block_log::npos) {
+                  block_file.seek(position);
+                  return read_serialized_block(block_file, size);
+               }
+               return retry_read_serialized_block_by_num(block_num);
+            }
+            FC_LOG_AND_RETHROW()
+         }
+
          std::optional<signed_block_header> read_block_header_by_num(uint32_t block_num) final {
             try {
                uint64_t pos = get_block_pos(block_num);
@@ -1032,6 +1100,16 @@ namespace eosio { namespace chain {
             return {};
          }
 
+         std::vector<char> retry_read_serialized_block_by_num(uint32_t block_num) final {
+            uint64_t block_size = 0;
+
+            auto ds = catalog.ro_stream_and_size_for_block(block_num, block_size);
+            if (ds) {
+               return read_serialized_block(*ds, block_size);
+            }
+            return {};
+         }
+
          std::optional<signed_block_header> retry_read_block_header_by_num(uint32_t block_num) final {
             auto ds = catalog.ro_stream_for_block(block_num);
             if (ds)
@@ -1230,6 +1308,11 @@ namespace eosio { namespace chain {
       return my->read_block_by_num(block_num);
    }
 
+   std::vector<char> block_log::read_serialized_block_by_num(uint32_t block_num) const {
+      std::lock_guard g(my->mtx);
+      return my->read_serialized_block_by_num(block_num);
+   }
+
    std::optional<signed_block_header> block_log::read_block_header_by_num(uint32_t block_num) const {
       std::lock_guard g(my->mtx);
       return my->read_block_header_by_num(block_num);

libraries/chain/controller.cpp

@@ -5404,6 +5404,14 @@ signed_block_ptr controller::fetch_block_by_number( uint32_t block_num )const  {
    return my->blog.read_block_by_num(block_num);
 } FC_CAPTURE_AND_RETHROW( (block_num) ) }
 
+std::vector<char> controller::fetch_serialized_block_by_number( uint32_t block_num)const  { try {
+   if (signed_block_ptr b = my->fork_db_fetch_block_on_best_branch_by_num(block_num)) {
+      return fc::raw::pack(*b);
+   }
+
+   return my->blog.read_serialized_block_by_num(block_num);
+} FC_CAPTURE_AND_RETHROW( (block_num) ) }
+
 std::optional<signed_block_header> controller::fetch_block_header_by_number( uint32_t block_num )const  { try {
    auto b = my->fork_db_fetch_block_on_best_branch_by_num(block_num);
    if (b)

libraries/chain/include/eosio/chain/block_log.hpp

@@ -53,7 +53,8 @@ namespace eosio { namespace chain {
          void reset( const genesis_state& gs, const signed_block_ptr& genesis_block );
          void reset( const chain_id_type& chain_id, uint32_t first_block_num );
 
-         signed_block_ptr read_block_by_num(uint32_t block_num)const;
+         signed_block_ptr  read_block_by_num(uint32_t block_num)const;
+         std::vector<char> read_serialized_block_by_num(uint32_t block_num)const;
          std::optional<signed_block_header> read_block_header_by_num(uint32_t block_num)const;
          std::optional<block_id_type>       read_block_id_by_num(uint32_t block_num)const;
 

libraries/chain/include/eosio/chain/controller.hpp

@@ -342,6 +342,8 @@ namespace eosio::chain {
          signed_block_ptr fetch_block_by_number( uint32_t block_num )const;
          // thread-safe
          signed_block_ptr fetch_block_by_id( const block_id_type& id )const;
+         // thread-safe, retrieves serialized signed block
+         std::vector<char> fetch_serialized_block_by_number( uint32_t block_num)const;
          // thread-safe
          bool block_exists(const block_id_type& id) const;
          bool validated_block_exists(const block_id_type& id) const;

libraries/chain/include/eosio/chain/log_catalog.hpp

@@ -31,6 +31,11 @@ struct null_verifier {
    void verify(const LogData&, const std::filesystem::path&) {}
 };
 
+struct block_pos_size_t {
+   uint64_t position = 0;  // start position of the block in block log
+   uint64_t size     = 0;  // size of the block
+};
+
 template <typename LogData, typename LogIndex, typename LogVerifier = null_verifier>
 struct log_catalog {
    using block_num_t = uint32_t;
@@ -159,6 +164,26 @@ struct log_catalog {
       }
    }
 
+   std::optional<block_pos_size_t> get_block_position_and_size(uint32_t block_num) {
+      std::optional<uint64_t> pos = get_block_position(block_num);
+
+      if (!pos) {
+         return {};
+      }
+
+      constexpr uint32_t block_pos_size = sizeof(uint64_t);
+      uint64_t block_size = 0;
+      assert(block_num <= log_data.last_block_num());
+      if (block_num < log_data.last_block_num()) {
+         uint64_t next_block_pos = log_index.nth_block_position(block_num + 1 - log_data.first_block_num());
+         block_size = next_block_pos - *pos - block_pos_size;
+      } else {
+         block_size = log_data.size() - *pos - block_pos_size;
+      }
+
+      return block_pos_size_t { .position = *pos, .size = block_size };
+   }
+
    fc::datastream<fc::cfile>* ro_stream_for_block(uint32_t block_num) {
       auto pos = get_block_position(block_num);
       if (pos) {
@@ -176,6 +201,15 @@ struct log_catalog {
       return {};
    }
 
+   fc::datastream<fc::cfile>* ro_stream_and_size_for_block(uint32_t block_num, uint64_t& block_size) {
+      auto pos_size = get_block_position_and_size(block_num);
+      if (pos_size) {
+         block_size = pos_size->size;
+         return &log_data.ro_stream_at(pos_size->position);
+      }
+      return nullptr;
+   }
+
    std::optional<block_id_type> id_for_block(uint32_t block_num) {
       auto pos = get_block_position(block_num);
       if (pos) {

libraries/state_history/abi.cpp

@@ -588,6 +588,13 @@ extern const char* const state_history_plugin_abi = R"({
                 { "type": "uint32", "name": "account_net_usage_average_window" }
             ]
         },
+        {
+            "name": "finalizer_authority", "fields": [
+                { "name": "description", "type": "string" },
+                { "name": "weight", "type": "uint64" },
+                { "name": "public_key", "type": "bytes" }
+            ]
+        },
         {
             "name": "finalizer_authority_with_string_key", "fields": [
                 { "name": "description", "type": "string" },
@@ -616,6 +623,55 @@ extern const char* const state_history_plugin_abi = R"({
                 { "name": "pending_finalizer_policy", "type": "finalizer_policy_with_string_key?" },
                 { "name": "last_pending_finalizer_policy_generation", "type": "uint32" }
             ]
+        },
+        {
+            "name": "protocol_feature_activation_extension", "fields": [
+                { "name": "protocol_features", "type": "checksum256[]" }
+            ]
+        },
+        {
+            "name": "producer_schedule_change_extension", "base": "producer_authority_schedule", "fields": []
+        },
+        {
+            "name": "qc_claim", "fields": [
+                { "name": "block_num", "type": "uint32" },
+                { "name": "is_strong_qc", "type": "bool" }
+            ]
+        },
+        {
+            "name": "insert_finalizer_policy_index_pair", "fields": [
+                { "name": "index", "type": "uint16" },
+                { "name": "value", "type": "finalizer_authority" }
+            ]
+        },
+        {
+            "name": "finalizer_policy_diff", "fields": [
+                { "name": "generation", "type": "uint32" },
+                { "name": "threshold", "type": "uint64" },
+                { "name": "remove_indexes", "type": "uint16[]" },
+                { "name": "insert_indexes", "type": "insert_finalizer_policy_index_pair[]" }
+            ]
+        },
+        {
+            "name": "insert_proposer_policy_index_pair", "fields": [
+                { "name": "index", "type": "uint16" },
+                { "name": "value", "type": "producer_authority" }
+            ]
+        },
+        {
+            "name": "proposer_policy_diff", "fields": [
+                { "name": "version", "type": "uint32" },
+                { "name": "proposal_time", "type": "block_timestamp_type" },
+                { "name": "remove_indexes", "type": "uint16[]" },
+                { "name": "insert_indexes", "type": "insert_proposer_policy_index_pair[]" }
+            ]
+        },
+        {
+            "name": "finality_extension", "fields": [
+                { "name": "qc_claim", "type": "qc_claim" },
+                { "name": "new_finalizer_policy_diff", "type": "finalizer_policy_diff?" },
+                { "name": "new_proposer_policy_diff", "type": "proposer_policy_diff?" }
+            ]
         }
     ],
     "types": [
@@ -657,7 +713,8 @@ extern const char* const state_history_plugin_abi = R"({
         { "name": "resource_limits_ratio", "types": ["resource_limits_ratio_v0"] },
         { "name": "elastic_limit_parameters", "types": ["elastic_limit_parameters_v0"] },
         { "name": "resource_limits_config", "types": ["resource_limits_config_v0"] },
-        { "name": "block_signing_authority", "types": ["block_signing_authority_v0"] }
+        { "name": "block_signing_authority", "types": ["block_signing_authority_v0"] },
+        { "name": "block_header_extension", "types": ["protocol_feature_activation_extension", "producer_schedule_change_extension", "finality_extension"] }
     ],
     "tables": [
         { "name": "account", "type": "account", "key_names": ["name"] },