Merge branch 'main' into finalizer_policies_in_finality_data

libraries/chain/block_header_state.cpp

@@ -67,6 +67,29 @@ const vector<digest_type>& block_header_state::get_new_protocol_feature_activati
    return detail::get_new_protocol_feature_activations(header_exts);
 }
 
+// The last proposed finalizer policy if none proposed or pending is the active finalizer policy
+const finalizer_policy& block_header_state::get_last_proposed_finalizer_policy() const {
+   if (!finalizer_policies.empty()) {
+      for (auto ritr = finalizer_policies.rbegin(); ritr != finalizer_policies.rend(); ++ritr) {
+         if (ritr->second.state == finalizer_policy_tracker::state_t::proposed)
+            return *ritr->second.policy;
+      }
+      return *finalizer_policies.rbegin()->second.policy;
+   }
+   return *active_finalizer_policy;
+}
+
+// The last proposed proposer policy, if none proposed then the active proposer policy
+const proposer_policy& block_header_state::get_last_proposed_proposer_policy() const {
+   if (proposer_policies.empty()) {
+      assert(active_proposer_policy);
+      return *active_proposer_policy;
+   }
+   auto it = proposer_policies.rbegin();
+   assert(it != proposer_policies.rend());
+   return *it->second;
+}
+
 // -------------------------------------------------------------------------------------------------
 // `finish_next` updates the next `block_header_state` according to the contents of the
 // header extensions (either new protocol_features or instant_finality_extension) applicable to this
@@ -103,10 +126,14 @@ void finish_next(const block_header_state& prev,
       }
    }
 
-   if (if_ext.new_proposer_policy) {
+   std::optional<proposer_policy> new_proposer_policy;
+   if (if_ext.new_proposer_policy_diff) {
+      new_proposer_policy = prev.get_last_proposed_proposer_policy().apply_diff(*if_ext.new_proposer_policy_diff);
+   }
+   if (new_proposer_policy) {
       // called when assembling the block
-      next_header_state.proposer_policies[if_ext.new_proposer_policy->active_time] =
-         std::move(if_ext.new_proposer_policy);
+      next_header_state.proposer_policies[new_proposer_policy->active_time] =
+         std::make_shared<proposer_policy>(std::move(*new_proposer_policy));
    }
 
    // finality_core
@@ -156,20 +183,21 @@ void finish_next(const block_header_state& prev,
       }
    }
 
-   if (if_ext.new_finalizer_policy) {
+   if (if_ext.new_finalizer_policy_diff) {
+      finalizer_policy new_finalizer_policy = prev.get_last_proposed_finalizer_policy().apply_diff(*if_ext.new_finalizer_policy_diff);
+
       // a new `finalizer_policy` was proposed in the previous block, and is present in the previous
       // block's header extensions.
       // Add this new proposal to the `finalizer_policies` multimap which tracks the in-flight proposals,
       // increment the generation number, and log that proposal (debug level).
       // ------------------------------------------------------------------------------------------------
-      dlog("New finalizer policy proposed in block ${id}: ${pol}",
-           ("id", prev.block_id)("pol", *if_ext.new_finalizer_policy));
-      next_header_state.finalizer_policy_generation = if_ext.new_finalizer_policy->generation;
+      dlog("New finalizer policy proposed in block ${id}..: ${pol}",
+           ("id", prev.block_id.str().substr(8,16))("pol", new_finalizer_policy));
+      next_header_state.finalizer_policy_generation = new_finalizer_policy.generation;
       next_header_state.finalizer_policies.emplace(
          next_header_state.block_num(),
          finalizer_policy_tracker{finalizer_policy_tracker::state_t::proposed,
-                                  std::make_shared<finalizer_policy>(std::move(*if_ext.new_finalizer_policy))});
-
+                                  std::make_shared<finalizer_policy>(std::move(new_finalizer_policy))});
    } else {
       next_header_state.finalizer_policy_generation = prev.finalizer_policy_generation;
    }
@@ -205,9 +233,17 @@ block_header_state block_header_state::next(block_header_state_input& input) con
 
    // finality extension
    // ------------------
+   std::optional<finalizer_policy_diff> new_finalizer_policy_diff;
+   if (input.new_finalizer_policy) {
+      new_finalizer_policy_diff = get_last_proposed_finalizer_policy().create_diff(*input.new_finalizer_policy);
+   }
+   std::optional<proposer_policy_diff> new_proposer_policy_diff;
+   if (input.new_proposer_policy) {
+      new_proposer_policy_diff = get_last_proposed_proposer_policy().create_diff(*input.new_proposer_policy);
+   }
    instant_finality_extension new_if_ext { input.most_recent_ancestor_with_qc,
-                                           std::move(input.new_finalizer_policy),
-                                           std::move(input.new_proposer_policy) };
+                                           std::move(new_finalizer_policy_diff),
+                                           std::move(new_proposer_policy_diff) };
 
    uint16_t if_ext_id = instant_finality_extension::extension_id();
    emplace_extension(next_header_state.header.header_extensions, if_ext_id, fc::raw::pack(new_if_ext));

libraries/chain/block_header_state_legacy.cpp

@@ -220,8 +220,10 @@ namespace eosio::chain {
          // set current block_num as qc_claim.last_qc_block_num in the IF extension
          qc_claim_t initial_if_claim { .block_num = block_num,
                                        .is_strong_qc = false };
+         finalizer_policy no_policy;
+         auto new_fin_policy_diff = no_policy.create_diff(*new_finalizer_policy);
          emplace_extension(h.header_extensions, instant_finality_extension::extension_id(),
-                           fc::raw::pack(instant_finality_extension{ initial_if_claim, std::move(new_finalizer_policy), {} }));
+                           fc::raw::pack(instant_finality_extension{ initial_if_claim, std::move(new_fin_policy_diff), {} }));
       } else if (qc_claim) {
          emplace_extension(h.header_extensions, instant_finality_extension::extension_id(),
                            fc::raw::pack(instant_finality_extension{ *qc_claim, {}, {} }));

libraries/chain/block_state.cpp

@@ -72,8 +72,8 @@ block_state_ptr block_state::create_if_genesis_block(const block_state_legacy& b
 
    assert(bsp.block->contains_header_extension(instant_finality_extension::extension_id())); // required by transition mechanism
    instant_finality_extension if_ext = bsp.block->extract_header_extension<instant_finality_extension>();
-   assert(if_ext.new_finalizer_policy); // required by transition mechanism
-   result.active_finalizer_policy = std::make_shared<finalizer_policy>(*if_ext.new_finalizer_policy);
+   assert(if_ext.new_finalizer_policy_diff); // required by transition mechanism
+   result.active_finalizer_policy = std::make_shared<finalizer_policy>(finalizer_policy{}.apply_diff(std::move(*if_ext.new_finalizer_policy_diff)));
    result.active_proposer_policy = std::make_shared<proposer_policy>();
    result.active_proposer_policy->active_time = bsp.timestamp();
    result.active_proposer_policy->proposer_schedule = bsp.active_schedule;

libraries/chain/controller.cpp

@@ -706,7 +706,7 @@ struct building_block {
    assembled_block assemble_block(boost::asio::io_context& ioc,
                                   const protocol_feature_set& pfs,
                                   fork_database& fork_db,
-                                  std::unique_ptr<proposer_policy> new_proposer_policy,
+                                  std::optional<proposer_policy> new_proposer_policy,
                                   std::optional<finalizer_policy> new_finalizer_policy,
                                   bool validating,
                                   std::optional<qc_data_t> validating_qc_data,
@@ -3164,13 +3164,13 @@ struct controller_impl {
          resource_limits.process_block_usage(bb.block_num());
 
          // Any proposer policy?
-         auto process_new_proposer_policy = [&](auto&) -> std::unique_ptr<proposer_policy> {
-            std::unique_ptr<proposer_policy> new_proposer_policy;
+         auto process_new_proposer_policy = [&](auto&) -> std::optional<proposer_policy> {
+            std::optional<proposer_policy> new_proposer_policy;
             const auto& gpo = db.get<global_property_object>();
             if (gpo.proposed_schedule_block_num) {
                std::optional<uint32_t> version = pending->get_next_proposer_schedule_version(gpo.proposed_schedule.producers);
                if (version) {
-                  new_proposer_policy                    = std::make_unique<proposer_policy>();
+                  new_proposer_policy.emplace();
                   new_proposer_policy->active_time       = detail::get_next_next_round_block_time(bb.timestamp());
                   new_proposer_policy->proposer_schedule = producer_authority_schedule::from_shared(gpo.proposed_schedule);
                   new_proposer_policy->proposer_schedule.version = *version;
@@ -3186,7 +3186,7 @@ struct controller_impl {
             }
             return new_proposer_policy;
          };
-         auto new_proposer_policy = apply_s<std::unique_ptr<proposer_policy>>(chain_head, process_new_proposer_policy);
+         auto new_proposer_policy = apply_s<std::optional<proposer_policy>>(chain_head, process_new_proposer_policy);
 
          // Any finalizer policy?
          std::optional<finalizer_policy> new_finalizer_policy = std::nullopt;
@@ -5283,6 +5283,8 @@ int64_t controller_impl::set_proposed_producers( vector<producer_authority> prod
    if (producers.empty())
       return -1; // INSTANT_FINALITY depends on DISALLOW_EMPTY_PRODUCER_SCHEDULE
 
+   EOS_ASSERT(producers.size() <= config::max_proposers, wasm_execution_error,
+              "Producer schedule exceeds the maximum proposer count for this chain");
    assert(pending);
 
    producer_authority_schedule sch;
@@ -5301,6 +5303,8 @@ int64_t controller_impl::set_proposed_producers( vector<producer_authority> prod
 }
 
 int64_t controller_impl::set_proposed_producers_legacy( vector<producer_authority> producers ) {
+   EOS_ASSERT(producers.size() <= config::max_producers, wasm_execution_error,
+              "Producer schedule exceeds the maximum producer count for this chain");
    const auto& gpo = db.get<global_property_object>();
    auto cur_block_num = chain_head.block_num() + 1;
 

libraries/chain/include/eosio/chain/block_header_state.hpp

@@ -46,8 +46,8 @@ struct finality_digest_data_v1 {
 // ------------------------------------------------------------------------------------------
 struct finalizer_policy_tracker {
    enum class state_t { proposed = 0, pending };
-   state_t               state;
-   finalizer_policy_ptr  policy;
+   state_t                    state;
+   finalizer_policy_ptr       policy;
 };
 
 struct building_block_input {
@@ -61,7 +61,7 @@ struct building_block_input {
 // this struct can be extracted from a building block
 struct block_header_state_input : public building_block_input {
    digest_type                       transaction_mroot;    // Comes from std::get<checksum256_type>(building_block::trx_mroot_or_receipt_digests)
-   std::shared_ptr<proposer_policy>  new_proposer_policy;  // Comes from building_block::new_proposer_policy
+   std::optional<proposer_policy>    new_proposer_policy;  // Comes from building_block::new_proposer_policy
    std::optional<finalizer_policy>   new_finalizer_policy; // Comes from building_block::new_finalizer_policy
    qc_claim_t                        most_recent_ancestor_with_qc; // Comes from traversing branch from parent and calling get_best_qc()
    digest_type                       finality_mroot_claim;
@@ -132,6 +132,9 @@ struct block_header_state {
 
    const vector<digest_type>& get_new_protocol_feature_activations() const;
    const producer_authority& get_scheduled_producer(block_timestamp_type t) const;
+
+   const finalizer_policy& get_last_proposed_finalizer_policy() const;
+   const proposer_policy& get_last_proposed_proposer_policy() const;
 };
 
 using block_header_state_ptr = std::shared_ptr<block_header_state>;

libraries/chain/include/eosio/chain/config.hpp

@@ -126,15 +126,16 @@ const static uint32_t   default_abi_serializer_max_time_us = 15*1000; ///< defau
  *  The number of sequential blocks produced by a single producer
  */
 const static int producer_repetitions = 12;
-const static int max_producers = 125;
+const static int max_producers = 125; // pre-savanna producer (proposer) limit
+const static int max_proposers = 64*1024; // savanna proposer (producer) limit
 
 const static size_t maximum_tracked_dpos_confirmations = 1024;     ///<
 static_assert(maximum_tracked_dpos_confirmations >= ((max_producers * 2 / 3) + 1) * producer_repetitions, "Settings never allow for DPOS irreversibility" );
 
 /**
  * Maximum number of finalizers in the finalizer set
  */
-const static size_t max_finalizers = 64*1024;
+const static size_t max_finalizers = 64*1024; // largest allowed finalizer policy diff
 const static size_t max_finalizer_description_size = 256;
 
 /**

libraries/chain/include/eosio/chain/finality/finalizer_policy.hpp

@@ -2,13 +2,41 @@
 
 #include <eosio/chain/types.hpp>
 #include <eosio/chain/finality/finalizer_authority.hpp>
+#include <fc/container/ordered_diff.hpp>
 
 namespace eosio::chain {
 
+   static_assert(std::numeric_limits<uint16_t>::max() >= config::max_finalizers - 1);
+   using finalizers_differ = fc::ordered_diff<finalizer_authority, uint16_t>;
+   using finalizers_diff_t = finalizers_differ::diff_result;
+
+   struct finalizer_policy_diff {
+      uint32_t                         generation = 0; ///< sequentially incrementing version number
+      uint64_t                         threshold = 0;  ///< vote weight threshold to finalize blocks
+      finalizers_diff_t                finalizers_diff;
+   };
+
    struct finalizer_policy {
       uint32_t                         generation = 0; ///< sequentially incrementing version number
       uint64_t                         threshold = 0;  ///< vote weight threshold to finalize blocks
-      std::vector<finalizer_authority> finalizers; ///< Instant Finality voter set
+      std::vector<finalizer_authority> finalizers;     ///< Instant Finality voter set
+
+      finalizer_policy_diff create_diff(const finalizer_policy& target) const {
+         return {.generation = target.generation,
+                 .threshold = target.threshold,
+                 .finalizers_diff = finalizers_differ::diff(finalizers, target.finalizers)};
+      }
+
+      template <typename X>
+      requires std::same_as<std::decay_t<X>, finalizer_policy_diff>
+      [[nodiscard]] finalizer_policy apply_diff(X&& diff) const {
+         finalizer_policy result;
+         result.generation = diff.generation;
+         result.threshold = diff.threshold;
+         auto copy = finalizers;
+         result.finalizers = finalizers_differ::apply_diff(std::move(copy), std::forward<X>(diff).finalizers_diff);
+         return result;
+      }
 
       // max accumulated weak weight before becoming weak_final
       uint64_t max_weak_sum_before_weak_final() const {
@@ -23,7 +51,10 @@ namespace eosio::chain {
    };
 
    using finalizer_policy_ptr = std::shared_ptr<finalizer_policy>;
+   using finalizer_policy_diff_ptr = std::shared_ptr<finalizer_policy_diff>;
 
 } /// eosio::chain
 
 FC_REFLECT( eosio::chain::finalizer_policy, (generation)(threshold)(finalizers) )
+FC_REFLECT( eosio::chain::finalizers_diff_t, (remove_indexes)(insert_indexes) )
+FC_REFLECT( eosio::chain::finalizer_policy_diff, (generation)(threshold)(finalizers_diff) )

libraries/chain/include/eosio/chain/finality/instant_finality_extension.hpp

@@ -12,11 +12,11 @@ struct instant_finality_extension : fc::reflect_init {
 
    instant_finality_extension() = default;
    instant_finality_extension(qc_claim_t qc_claim,
-                              std::optional<finalizer_policy> new_finalizer_policy,
-                              std::shared_ptr<proposer_policy> new_proposer_policy) :
+                              std::optional<finalizer_policy_diff>&& new_finalizer_policy_diff,
+                              std::optional<proposer_policy_diff>&& new_proposer_policy_diff) :
       qc_claim(qc_claim),
-      new_finalizer_policy(std::move(new_finalizer_policy)),
-      new_proposer_policy(std::move(new_proposer_policy))
+      new_finalizer_policy_diff(std::move(new_finalizer_policy_diff)),
+      new_proposer_policy_diff(std::move(new_proposer_policy_diff))
    {}
 
    void reflector_init() const {
@@ -25,11 +25,11 @@ struct instant_finality_extension : fc::reflect_init {
       static_assert( extension_id() == 2, "instant_finality_extension extension id must be 2" );
    }
 
-   qc_claim_t                         qc_claim;
-   std::optional<finalizer_policy>    new_finalizer_policy;
-   std::shared_ptr<proposer_policy>   new_proposer_policy;
+   qc_claim_t                              qc_claim;
+   std::optional<finalizer_policy_diff>    new_finalizer_policy_diff;
+   std::optional<proposer_policy_diff>     new_proposer_policy_diff;
 };
 
 } /// eosio::chain
 
-FC_REFLECT( eosio::chain::instant_finality_extension, (qc_claim)(new_finalizer_policy)(new_proposer_policy) )
+FC_REFLECT( eosio::chain::instant_finality_extension, (qc_claim)(new_finalizer_policy_diff)(new_proposer_policy_diff) )

libraries/chain/include/eosio/chain/finality/proposer_policy.hpp

@@ -5,16 +5,44 @@
 
 namespace eosio::chain {
 
+static_assert(std::numeric_limits<uint16_t>::max() >= config::max_proposers - 1);
+using producer_auth_differ = fc::ordered_diff<producer_authority, uint16_t>;
+using producer_auth_diff_t = producer_auth_differ::diff_result;
+
+struct proposer_policy_diff {
+   uint32_t                    version = 0; ///< sequentially incrementing version number of producer_authority_schedule
+   block_timestamp_type        active_time; // block when schedule will become active
+   producer_auth_diff_t        producer_auth_diff;
+};
+
 struct proposer_policy {
-   constexpr static uint8_t    current_schema_version = 1;
-   uint8_t                     schema_version {current_schema_version};
    // Useful for light clients, not necessary for nodeos
    block_timestamp_type        active_time; // block when schedule will become active
    producer_authority_schedule proposer_schedule;
+
+   proposer_policy_diff create_diff(const proposer_policy& target) const {
+      return {.version = target.proposer_schedule.version,
+              .active_time = target.active_time,
+              .producer_auth_diff = producer_auth_differ::diff(proposer_schedule.producers, target.proposer_schedule.producers)};
+   }
+
+   template <typename X>
+   requires std::same_as<std::decay_t<X>, proposer_policy_diff>
+   [[nodiscard]] proposer_policy apply_diff(X&& diff) const {
+      proposer_policy result;
+      result.proposer_schedule.version = diff.version;
+      result.active_time = diff.active_time;
+      auto copy = proposer_schedule.producers;
+      result.proposer_schedule.producers = producer_auth_differ::apply_diff(std::move(copy),
+                                                                            std::forward<X>(diff).producer_auth_diff);
+      return result;
+   }
 };
 
 using proposer_policy_ptr = std::shared_ptr<proposer_policy>;
 
 } /// eosio::chain
 
-FC_REFLECT( eosio::chain::proposer_policy, (schema_version)(active_time)(proposer_schedule) )
+FC_REFLECT( eosio::chain::proposer_policy, (active_time)(proposer_schedule) )
+FC_REFLECT( eosio::chain::producer_auth_diff_t, (remove_indexes)(insert_indexes) )
+FC_REFLECT( eosio::chain::proposer_policy_diff, (version)(active_time)(producer_auth_diff) )

libraries/chain/webassembly/privileged.cpp

@@ -44,7 +44,6 @@ namespace eosio { namespace chain { namespace webassembly {
    }
 
    int64_t set_proposed_producers_common( apply_context& context, vector<producer_authority> && producers, bool validate_keys ) {
-      EOS_ASSERT(producers.size() <= config::max_producers, wasm_execution_error, "Producer schedule exceeds the maximum producer count for this chain");
       EOS_ASSERT( producers.size() > 0
                   || !context.control.is_builtin_activated( builtin_protocol_feature_t::disallow_empty_producer_schedule ),
                   wasm_execution_error,