hotshot update (#1866)

Bumps hotshot to 0.1.70

  * implement `Versions`
  * propagate related (many) changes
  * use `UpdateLock` for Message (de)serialization
  * bump upstream crates

---------

Co-authored-by: tbro <[email protected]>
Co-authored-by: Theodore Schnepper <[email protected]>

Cargo.toml

@@ -52,22 +52,22 @@ dotenvy = "0.15"
 ethers = { version = "2.0", features = ["solc"] }
 futures = "0.3"
 
-hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.69" }
+hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.70" }
 # Hotshot imports
-hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.69" }
-hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "0.1.40" }
-marketplace-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", tag = "0.0.3" }
-hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.40" }
-hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.69" }
-hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.51" }
-hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.69" }
+hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.70" }
+hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "0.1.41" }
+marketplace-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", tag = "0.0.5" }
+hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.41" }
+hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.70" }
+hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.52" }
+hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.70" }
 hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" }
-hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.69" }
-hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.69" }
-hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.69" }
+hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.70" }
+hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.70" }
+hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.70" }
 hotshot-contract-adapter = { version = "0.1.0", path = "contracts/rust/adapter" }
 # Temporary, used to pull in the mock auction results provider
-hotshot-example-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.69" }
+hotshot-example-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.70" }
 
 # Push CDN imports
 cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", features = [

builder/src/lib.rs

@@ -121,7 +121,7 @@ pub mod testing {
 
     use espresso_types::{
         mock::MockStateCatchup, v0_3::ChainConfig, Event, FeeAccount, L1Client, NodeState, PrivKey,
-        PubKey, Transaction, ValidatedState,
+        PubKey, SequencerVersions, Transaction, ValidatedState,
     };
     use ethers::{
         types::spoof::State,
@@ -339,7 +339,7 @@ pub mod testing {
             bind_version: Ver,
             options: impl PersistenceOptions<Persistence = P>,
         ) -> Vec<(
-            Arc<SystemContextHandle<SeqTypes, Node<network::Memory, P>>>,
+            Arc<SystemContextHandle<SeqTypes, Node<network::Memory, P>, SequencerVersions>>,
             Option<StateSigner<Ver>>,
         )> {
             let num_staked_nodes = self.num_staked_nodes();
@@ -380,7 +380,7 @@ pub mod testing {
             bind_version: Ver,
             persistence: P,
         ) -> (
-            SystemContextHandle<SeqTypes, Node<network::Memory, P>>,
+            SystemContextHandle<SeqTypes, Node<network::Memory, P>, SequencerVersions>,
             StateSigner<Ver>,
         ) {
             let mut config = self.config.clone();
@@ -465,7 +465,9 @@ pub mod testing {
             hotshot_events_api_url: Url,
             known_nodes_with_stake: Vec<PeerConfig<VerKey>>,
             num_non_staking_nodes: usize,
-            hotshot_context_handle: Arc<SystemContextHandle<SeqTypes, Node<network::Memory, P>>>,
+            hotshot_context_handle: Arc<
+                SystemContextHandle<SeqTypes, Node<network::Memory, P>, SequencerVersions>,
+            >,
         ) {
             // create a event streamer
             let events_streamer = Arc::new(RwLock::new(EventsStreamer::new(
@@ -602,7 +604,9 @@ pub mod testing {
     {
         pub async fn init_permissioned_builder(
             hotshot_test_config: HotShotTestConfig,
-            hotshot_handle: Arc<SystemContextHandle<SeqTypes, Node<network::Memory, P>>>,
+            hotshot_handle: Arc<
+                SystemContextHandle<SeqTypes, Node<network::Memory, P>, SequencerVersions>,
+            >,
             node_id: u64,
             state_signer: StateSigner<Ver>,
             hotshot_builder_api_url: Url,

builder/src/non_permissioned.rs

@@ -11,7 +11,7 @@ use async_compatibility_layer::{
 use async_std::sync::{Arc, RwLock};
 use espresso_types::{
     eth_signature_key::EthKeyPair, v0_3::ChainConfig, FeeAmount, L1Client, NodeState, Payload,
-    SeqTypes, ValidatedState,
+    SeqTypes, SequencerVersions, ValidatedState,
 };
 use ethers::{
     core::k256::ecdsa::SigningKey,
@@ -195,14 +195,18 @@ impl BuilderConfig {
         let events_url = hotshot_events_api_url.clone();
         tracing::info!("Running permissionless builder against hotshot events API at {events_url}",);
         async_spawn(async move {
-            let res = run_non_permissioned_standalone_builder_service(
-                da_sender,
-                qc_sender,
-                decide_sender,
-                tx_sender,
-                events_url,
-            )
-            .await;
+            // TODO this is proabably a mistake in builder-core. These
+            // generic params can be removed and the concrete types
+            // used instead.
+            let res =
+                run_non_permissioned_standalone_builder_service::<SeqTypes, SequencerVersions>(
+                    da_sender,
+                    qc_sender,
+                    decide_sender,
+                    tx_sender,
+                    events_url,
+                )
+                .await;
             tracing::error!(?res, "builder service exited");
             if res.is_err() {
                 panic!("Builder should restart.");

builder/src/permissioned.rs

@@ -26,8 +26,8 @@ use async_std::{
 use espresso_types::{
     eth_signature_key::EthKeyPair,
     v0::traits::{PersistenceOptions, SequencerPersistence, StateCatchup},
-    FeeAmount, L1Client, NodeState, Payload, PubKey, SeqTypes, SolverAuctionResultsProvider,
-    ValidatedState,
+    FeeAmount, L1Client, NodeState, Payload, PubKey, SeqTypes, SequencerVersions,
+    SolverAuctionResultsProvider, ValidatedState,
 };
 use ethers::{
     core::k256::ecdsa::SigningKey,
@@ -339,7 +339,10 @@ pub async fn init_hotshot<
     stake_table_commit: StakeTableCommitmentType,
     _: Ver,
     persistence: P,
-) -> (SystemContextHandle<SeqTypes, Node<N, P>>, StateSigner<Ver>) {
+) -> (
+    SystemContextHandle<SeqTypes, Node<N, P>, SequencerVersions>,
+    StateSigner<Ver>,
+) {
     let combined_known_nodes_with_stake = match stake_table_entries_for_non_voting_nodes {
         Some(stake_table_entries) => {
             let combined_entries = config

hotshot-state-prover/src/bin/state-prover.rs

@@ -1,15 +1,15 @@
 use std::time::Duration;
 
 use clap::Parser;
-use espresso_types::{parse_duration, SeqTypes};
+use espresso_types::{parse_duration, SequencerVersions};
 use ethers::{
     providers::{Http, Middleware, Provider},
     signers::{coins_bip39::English, MnemonicBuilder, Signer},
     types::Address,
 };
 use hotshot_stake_table::config::STAKE_TABLE_CAPACITY;
 use hotshot_state_prover::service::{run_prover_once, run_prover_service, StateProverConfig};
-use hotshot_types::traits::node_implementation::NodeType;
+use hotshot_types::traits::node_implementation::Versions;
 use sequencer_utils::logging;
 use url::Url;
 use vbs::version::StaticVersionType;
@@ -114,13 +114,16 @@ async fn main() {
 
     if args.daemon {
         // Launching the prover service daemon
-        if let Err(err) = run_prover_service(config, <SeqTypes as NodeType>::Base::instance()).await
+        if let Err(err) =
+            run_prover_service(config, <SequencerVersions as Versions>::Base::instance()).await
         {
             tracing::error!("Error running prover service: {:?}", err);
         };
     } else {
         // Run light client state update once
-        if let Err(err) = run_prover_once(config, <SeqTypes as NodeType>::Base::instance()).await {
+        if let Err(err) =
+            run_prover_once(config, <SequencerVersions as Versions>::Base::instance()).await
+        {
             tracing::error!("Error running prover once: {:?}", err);
         };
     }

marketplace-builder/src/bin/marketplace-builder.rs

@@ -3,15 +3,17 @@ use std::{num::NonZeroUsize, path::PathBuf, time::Duration};
 use async_compatibility_layer::logging::{setup_backtrace, setup_logging};
 use clap::Parser;
 use espresso_types::{
-    eth_signature_key::EthKeyPair, parse_duration, FeeAmount, NamespaceId, SeqTypes,
+    eth_signature_key::EthKeyPair, parse_duration, FeeAmount, NamespaceId, SequencerVersions,
 };
 use hotshot::traits::ValidatedState;
-use hotshot_types::{data::ViewNumber, traits::node_implementation::ConsensusTime};
+use hotshot_types::{
+    data::ViewNumber,
+    traits::node_implementation::{ConsensusTime, Versions},
+};
 use marketplace_builder::{
     builder::{build_instance_state, BuilderConfig},
     hooks::BidConfig,
 };
-use marketplace_builder_core::testing::basic_test::NodeType;
 use sequencer::{Genesis, L1Params};
 use url::Url;
 use vbs::version::StaticVersionType;
@@ -150,7 +152,7 @@ async fn main() -> anyhow::Result<()> {
         genesis.chain_config,
         l1_params,
         opt.state_peers,
-        <SeqTypes as NodeType>::Base::instance(),
+        <SequencerVersions as Versions>::Base::instance(),
     )
     .unwrap();
 

marketplace-builder/src/builder.rs

@@ -12,7 +12,7 @@ use async_lock::RwLock;
 use async_std::sync::Arc;
 use espresso_types::{
     eth_signature_key::EthKeyPair, v0_3::ChainConfig, FeeAmount, L1Client, NamespaceId, NodeState,
-    Payload, SeqTypes, ValidatedState,
+    Payload, SeqTypes, SequencerVersions, ValidatedState,
 };
 use ethers::{
     core::k256::ecdsa::SigningKey,
@@ -203,9 +203,11 @@ impl BuilderConfig {
             };
 
             async_spawn(async move {
-                let res =
-                    run_non_permissioned_standalone_builder_service(hooks, senders, events_url)
-                        .await;
+                let res = run_non_permissioned_standalone_builder_service::<
+                    SeqTypes,
+                    SequencerVersions,
+                >(hooks, senders, events_url)
+                .await;
                 tracing::error!(?res, "Reserve builder service exited");
                 if res.is_err() {
                     panic!("Reserve builder should restart.");
@@ -217,9 +219,11 @@ impl BuilderConfig {
             let hooks = hooks::EspressoFallbackHooks { solver_api_url };
 
             async_spawn(async move {
-                let res =
-                    run_non_permissioned_standalone_builder_service(hooks, senders, events_url)
-                        .await;
+                let res = run_non_permissioned_standalone_builder_service::<
+                    SeqTypes,
+                    SequencerVersions,
+                >(hooks, senders, events_url)
+                .await;
                 tracing::error!(?res, "Fallback builder service exited");
                 if res.is_err() {
                     panic!("Fallback builder should restart.");
@@ -249,7 +253,8 @@ mod test {
     use async_std::{stream::StreamExt, task};
     use committable::Commitment;
     use espresso_types::{
-        mock::MockStateCatchup, FeeAccount, NamespaceId, PubKey, SeqTypes, Transaction,
+        mock::MockStateCatchup, FeeAccount, NamespaceId, PubKey, SeqTypes, SequencerVersions,
+        Transaction,
     };
     use ethers::utils::Anvil;
     use hotshot::types::BLSPrivKey;
@@ -261,12 +266,11 @@ mod test {
     use hotshot_query_service::availability::LeafQueryData;
     use hotshot_types::{
         bundle::Bundle,
-        constants::MarketplaceVersion,
         light_client::StateKeyPair,
         signature_key::BLSPubKey,
         traits::{
             block_contents::{BlockPayload, GENESIS_VID_NUM_STORAGE_NODES},
-            node_implementation::NodeType,
+            node_implementation::{NodeType, Versions},
             signature_key::{BuilderSignatureKey, SignatureKey},
         },
     };
@@ -325,7 +329,8 @@ mod test {
             )
             .network_config(network_config)
             .build();
-        let network = TestNetwork::new(config, <SeqTypes as NodeType>::Base::instance()).await;
+        let network =
+            TestNetwork::new(config, <SequencerVersions as Versions>::Base::instance()).await;
 
         // Start the builder
         let init = BuilderConfig::init(
@@ -352,7 +357,7 @@ mod test {
         let _builder_config = init.await;
 
         // Wait for at least one empty block to be sequenced (after consensus starts VID).
-        let sequencer_client: Client<ServerError, <SeqTypes as NodeType>::Base> =
+        let sequencer_client: Client<ServerError, <SequencerVersions as Versions>::Base> =
             Client::new(query_api_url);
         sequencer_client.connect(None).await;
         sequencer_client
@@ -366,12 +371,12 @@ mod test {
             .unwrap();
 
         //  Connect to builder
-        let builder_client: Client<ServerError, MarketplaceVersion> =
+        let builder_client: Client<ServerError, <SequencerVersions as Versions>::Marketplace> =
             Client::new(builder_api_url.clone());
         builder_client.connect(None).await;
 
         //  TODO(AG): workaround for version mismatch between bundle and submit APIs
-        let submission_client: Client<ServerError, <SeqTypes as NodeType>::Base> =
+        let submission_client: Client<ServerError, <SequencerVersions as Versions>::Base> =
             Client::new(builder_api_url);
         submission_client.connect(None).await;
 

marketplace-builder/src/hooks.rs

@@ -5,10 +5,12 @@ use espresso_types::v0_3::BidTxBody;
 
 use espresso_types::v0_3::RollupRegistration;
 use espresso_types::SeqTypes;
+use espresso_types::SequencerVersions;
 use hotshot::types::EventType;
 
 use hotshot::types::Event;
 
+use hotshot_types::traits::node_implementation::Versions;
 use marketplace_builder_core::service::BuilderHooks;
 
 use espresso_types::FeeAmount;
@@ -36,8 +38,8 @@ pub struct BidConfig {
 
 pub async fn connect_to_solver(
     solver_api_url: Url,
-) -> Option<Client<SolverError, <SeqTypes as NodeType>::Base>> {
-    let client = Client::<SolverError, <SeqTypes as NodeType>::Base>::new(
+) -> Option<Client<SolverError, <SequencerVersions as Versions>::Base>> {
+    let client = Client::<SolverError, <SequencerVersions as Versions>::Base>::new(
         solver_api_url.join("marketplace-solver/").unwrap(),
     );
 

marketplace-builder/src/lib.rs

@@ -19,7 +19,7 @@ use espresso_types::{
     eth_signature_key::EthKeyPair,
     v0::traits::{PersistenceOptions, SequencerPersistence, StateCatchup},
     v0_3::BidTxBody,
-    SeqTypes,
+    SeqTypes, SequencerVersions,
 };
 use ethers::{
     core::k256::ecdsa::SigningKey,
@@ -56,7 +56,7 @@ use hotshot_types::{
         },
         election::Membership,
         metrics::Metrics,
-        node_implementation::{ConsensusTime, NodeType},
+        node_implementation::{ConsensusTime, NodeType, Versions},
     },
     utils::BuilderCommitment,
     HotShotConfig, PeerConfig, ValidatorConfig,
@@ -92,7 +92,7 @@ pub fn run_builder_api_service(url: Url, source: ProxyGlobalState<SeqTypes>) {
     let private_mempool_api = hotshot_builder_api::v0_3::builder::submit_api::<
         ProxyGlobalState<SeqTypes>,
         SeqTypes,
-        <SeqTypes as NodeType>::Base,
+        <SequencerVersions as Versions>::Base,
     >(&HotshotBuilderApiOptions::default())
     .expect("Failed to construct the builder API for private mempool txns");
 
@@ -104,5 +104,5 @@ pub fn run_builder_api_service(url: Url, source: ProxyGlobalState<SeqTypes>) {
     app.register_module("txn_submit", private_mempool_api)
         .expect("Failed to register the private mempool API");
 
-    async_spawn(app.serve(url, <SeqTypes as NodeType>::Base::instance()));
+    async_spawn(app.serve(url, <SequencerVersions as Versions>::Base::instance()));
 }

marketplace-solver/src/events.rs

@@ -2,21 +2,22 @@ use std::{pin::Pin, sync::Arc};
 
 use anyhow::Context;
 use async_std::sync::RwLock;
-use espresso_types::SeqTypes;
+use espresso_types::{SeqTypes, SequencerVersions};
 use futures::{Stream, StreamExt as _};
 use hotshot::types::Event;
 use hotshot_events_service::{events, events_source::StartupInfo};
-use hotshot_types::traits::node_implementation::NodeType;
+use hotshot_types::traits::node_implementation::Versions;
 use surf_disco::Client;
 use tide_disco::Url;
 
 use crate::state::GlobalState;
 
-pub struct EventsServiceClient(Client<events::Error, <SeqTypes as NodeType>::Base>);
+pub struct EventsServiceClient(Client<events::Error, <SequencerVersions as Versions>::Base>);
 
 impl EventsServiceClient {
     pub async fn new(url: Url) -> Self {
-        let client = Client::<events::Error, <SeqTypes as NodeType>::Base>::new(url.clone());
+        let client =
+            Client::<events::Error, <SequencerVersions as Versions>::Base>::new(url.clone());
 
         client.connect(None).await;