[WEEKLY RELEASE] HotShot 0.5.57

Cargo.toml

@@ -47,18 +47,18 @@ dotenvy = "0.15"
 ethers = { version = "2.0", features = ["solc"] }
 futures = "0.3"
 
-hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" }
+hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" }
 # Hotshot imports
-hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "rc-0.5.56" }
-hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "0.1.25" }
-hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.25" }
-hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" }
-hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.34" }
-hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" }
+hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "rc-0.5.57" }
+hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "0.1.27" }
+hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.27" }
+hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" }
+hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.37" }
+hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" }
 hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" }
-hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" }
-hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" }
-hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" }
+hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" }
+hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" }
+hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" }
 hotshot-contract-adapter = { version = "0.1.0", path = "contracts/rust/adapter" }
 
 # Push CDN imports
@@ -99,10 +99,10 @@ jf-relation = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4
 jf-utils = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.5" }
 snafu = "0.8"
 strum = { version = "0.26", features = ["derive"] }
-surf-disco = "0.7"
+surf-disco = "0.8"
 tagged-base64 = "0.4"
+tide-disco = "0.8"
 thiserror = "1.0.61"
-tide-disco = "0.7"
 time = "0.3"
 tracing = "0.1"
 bytesize = "1.3"

sequencer/src/api/endpoints.rs

@@ -108,7 +108,7 @@ where
                 )
                 .context(CustomSnafu {
                     message: format!("failed to make proof for namespace {ns_id}"),
-                    status: StatusCode::NotFound,
+                    status: StatusCode::NOT_FOUND,
                 })?;
 
             let transactions = if let NamespaceProof::Existence {
@@ -184,7 +184,7 @@ where
             if tx.namespace() > NamespaceId::from(u32::MAX as u64) {
                 return Err(Error::Custom {
                     message: "Transaction namespace > u32::MAX".to_string(),
-                    status: StatusCode::BadRequest,
+                    status: StatusCode::BAD_REQUEST,
                 });
             }
 
@@ -221,7 +221,7 @@ where
                 .get_state_signature(height)
                 .await
                 .ok_or(tide_disco::Error::catch_all(
-                    StatusCode::NotFound,
+                    StatusCode::NOT_FOUND,
                     "Signature not found.".to_owned(),
                 ))
         }
@@ -252,15 +252,15 @@ where
                 .map_err(Error::from_request_error)?;
             let account = account.parse().map_err(|err| {
                 Error::catch_all(
-                    StatusCode::BadRequest,
+                    StatusCode::BAD_REQUEST,
                     format!("malformed account {account}: {err}"),
                 )
             })?;
 
             state
                 .get_account(height, ViewNumber::new(view), account)
                 .await
-                .map_err(|err| Error::catch_all(StatusCode::NotFound, format!("{err:#}")))
+                .map_err(|err| Error::catch_all(StatusCode::NOT_FOUND, format!("{err:#}")))
         }
         .boxed()
     })?
@@ -276,7 +276,7 @@ where
             state
                 .get_frontier(height, ViewNumber::new(view))
                 .await
-                .map_err(|err| Error::catch_all(StatusCode::NotFound, format!("{err:#}")))
+                .map_err(|err| Error::catch_all(StatusCode::NOT_FOUND, format!("{err:#}")))
         }
         .boxed()
     })?;
@@ -314,7 +314,7 @@ where
     let mut api = Api::<S, Error, Ver>::new(toml)?;
 
     let env_variables = get_public_env_vars()
-        .map_err(|err| Error::catch_all(StatusCode::InternalServerError, format!("{err:#}")))?;
+        .map_err(|err| Error::catch_all(StatusCode::INTERNAL_SERVER_ERROR, format!("{err:#}")))?;
 
     api.get("hotshot", |_, state| {
         async move { Ok(state.get_config().await) }.boxed()

sequencer/src/bin/nasty-client.rs

@@ -510,7 +510,7 @@ impl<T: Queryable> ResourceManager<T> {
         let elapsed = start.elapsed();
 
         let status = match &res {
-            Ok(_) => StatusCode::Ok,
+            Ok(_) => StatusCode::OK,
             Err(err) => err.status(),
         };
         tracing::debug!("<- GET {path} {} ({elapsed:?})", u16::from(status));

sequencer/src/hotshot_commitment.rs

@@ -320,7 +320,7 @@ mod test {
                 .flatten()
                 .ok_or_else(|| {
                     Self::Error::catch_all(
-                        StatusCode::NotFound,
+                        StatusCode::NOT_FOUND,
                         format!("no leaf for height {height}"),
                     )
                 })

sequencer/src/lib.rs

@@ -54,7 +54,7 @@ use hotshot_orchestrator::{
 };
 use hotshot_types::{
     consensus::CommitmentMap,
-    data::{DaProposal, VidDisperseShare, ViewNumber},
+    data::{DaProposal, QuorumProposal, VidDisperseShare, ViewNumber},
     event::HotShotAction,
     light_client::{StateKeyPair, StateSignKey},
     message::Proposal,
@@ -162,6 +162,13 @@ impl<P: SequencerPersistence> Storage<SeqTypes> for Arc<RwLock<P>> {
             .update_undecided_state(leaves, state)
             .await
     }
+
+    async fn append_proposal(
+        &self,
+        proposal: &Proposal<SeqTypes, QuorumProposal<SeqTypes>>,
+    ) -> anyhow::Result<()> {
+        self.write().await.append_quorum_proposal(proposal).await
+    }
 }
 
 #[derive(Debug, Clone)]

sequencer/src/persistence.rs

@@ -20,7 +20,7 @@ use hotshot::{
 };
 use hotshot_types::{
     consensus::CommitmentMap,
-    data::{DaProposal, VidDisperseShare},
+    data::{DaProposal, QuorumProposal, VidDisperseShare},
     event::{HotShotAction, LeafInfo},
     message::Proposal,
     simple_certificate::QuorumCertificate,
@@ -87,6 +87,11 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static {
         &self,
     ) -> anyhow::Result<Option<(CommitmentMap<Leaf>, BTreeMap<ViewNumber, View<SeqTypes>>)>>;
 
+    /// Load the proposals saved by consensus
+    async fn load_quorum_proposals(
+        &self,
+    ) -> anyhow::Result<Option<BTreeMap<ViewNumber, Proposal<SeqTypes, QuorumProposal<SeqTypes>>>>>;
+
     async fn load_vid_share(
         &self,
         view: ViewNumber,
@@ -168,20 +173,29 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static {
             .context("loading undecided state")?
             .unwrap_or_default();
 
+        let saved_proposals = self
+            .load_quorum_proposals()
+            .await
+            .context("loading saved proposals")
+            .unwrap_or_default()
+            .unwrap_or_default();
+
         tracing::info!(
             ?leaf,
             ?view,
             ?high_qc,
             ?validated_state,
             ?undecided_leaves,
             ?undecided_state,
+            ?saved_proposals,
             "loaded consensus state"
         );
         Ok(HotShotInitializer::from_reload(
             leaf,
             state,
             validated_state,
             view,
+            saved_proposals,
             high_qc,
             undecided_leaves.into_values().collect(),
             undecided_state,
@@ -233,6 +247,10 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static {
         leaves: CommitmentMap<Leaf>,
         state: BTreeMap<ViewNumber, View<SeqTypes>>,
     ) -> anyhow::Result<()>;
+    async fn append_quorum_proposal(
+        &mut self,
+        proposal: &Proposal<SeqTypes, QuorumProposal<SeqTypes>>,
+    ) -> anyhow::Result<()>;
 }
 
 #[cfg(test)]

sequencer/src/persistence/fs.rs

@@ -6,7 +6,7 @@ use clap::Parser;
 
 use hotshot_types::{
     consensus::CommitmentMap,
-    data::{DaProposal, VidDisperseShare},
+    data::{DaProposal, QuorumProposal, VidDisperseShare},
     event::HotShotAction,
     message::Proposal,
     simple_certificate::QuorumCertificate,
@@ -99,6 +99,10 @@ impl Persistence {
         self.path.join("undecided_state")
     }
 
+    fn proposals_dir_path(&self) -> PathBuf {
+        self.path.join("proposals")
+    }
+
     /// Overwrite a file if a condition is met.
     ///
     /// The file at `path`, if it exists, is opened in read mode and passed to `pred`. If `pred`
@@ -408,6 +412,86 @@ impl SequencerPersistence for Persistence {
             },
         )
     }
+    async fn append_quorum_proposal(
+        &mut self,
+        proposal: &Proposal<SeqTypes, QuorumProposal<SeqTypes>>,
+    ) -> anyhow::Result<()> {
+        let view_number = proposal.data.view_number().u64();
+        let dir_path = self.proposals_dir_path();
+
+        fs::create_dir_all(dir_path.clone()).context("failed to create proposals dir")?;
+
+        let file_path = dir_path.join(view_number.to_string()).with_extension("txt");
+        self.replace(
+            &file_path,
+            |_| {
+                // Don't overwrite the file, we want to append since we can get multiple
+                // saved quorum proposals.
+                Ok(false)
+            },
+            |mut file| {
+                let proposal_bytes = bincode::serialize(&proposal).context("serialize proposal")?;
+
+                file.write_all(&proposal_bytes)?;
+                Ok(())
+            },
+        )
+    }
+    async fn load_quorum_proposals(
+        &self,
+    ) -> anyhow::Result<Option<BTreeMap<ViewNumber, Proposal<SeqTypes, QuorumProposal<SeqTypes>>>>>
+    {
+        // First, get the proposal directory.
+        let dir_path = self.proposals_dir_path();
+
+        // Then, we want to get the entries in this directory since they'll be the
+        // key/value pairs for our map.
+        let files: Vec<fs::DirEntry> = fs::read_dir(dir_path.clone())?
+            .filter_map(|entry| {
+                entry
+                    .ok()
+                    .filter(|e| e.file_type().map(|ft| ft.is_file()).unwrap_or(false))
+            })
+            .collect();
+
+        // Do we have any entries?
+        if files.is_empty() {
+            // Don't both continuing if we don't have any data.
+            return Ok(None);
+        }
+
+        // Read all of the files
+        let proposal_files = files
+            .into_iter()
+            .map(|entry| dir_path.join(entry.file_name()).with_extension("txt"));
+
+        let mut map = BTreeMap::new();
+        for file in proposal_files.into_iter() {
+            // This operation shouldn't fail, but we don't want to panic here if the filesystem
+            // somehow gets corrupted. We get the stem to remove the ".txt" from the end.
+            if let Some(file_name) = file.file_stem() {
+                // We need to convert the filename (which corresponds to the view)
+                let view_number = ViewNumber::new(
+                    file_name
+                        .to_string_lossy()
+                        .parse::<u64>()
+                        .context("convert file name to u64")?,
+                );
+
+                // Now, we'll try and load the proposal associated with this function.
+                let proposal_bytes = fs::read(file)?;
+
+                // Then, deserialize.
+                let proposal: Proposal<SeqTypes, QuorumProposal<SeqTypes>> =
+                    bincode::deserialize(&proposal_bytes)?;
+
+                // Push to the map and we're done.
+                map.insert(view_number, proposal);
+            }
+        }
+
+        Ok(Some(map))
+    }
 }
 
 #[cfg(test)]

sequencer/src/persistence/no_storage.rs

@@ -6,7 +6,7 @@ use crate::{Leaf, SeqTypes, ViewNumber};
 use async_trait::async_trait;
 use hotshot_types::{
     consensus::CommitmentMap,
-    data::{DaProposal, VidDisperseShare},
+    data::{DaProposal, QuorumProposal, VidDisperseShare},
     event::HotShotAction,
     message::Proposal,
     simple_certificate::QuorumCertificate,
@@ -85,6 +85,13 @@ impl SequencerPersistence for NoStorage {
         Ok(None)
     }
 
+    async fn load_quorum_proposals(
+        &self,
+    ) -> anyhow::Result<Option<BTreeMap<ViewNumber, Proposal<SeqTypes, QuorumProposal<SeqTypes>>>>>
+    {
+        Ok(None)
+    }
+
     async fn append_vid(
         &mut self,
         _proposal: &Proposal<SeqTypes, VidDisperseShare<SeqTypes>>,
@@ -111,4 +118,10 @@ impl SequencerPersistence for NoStorage {
     ) -> anyhow::Result<()> {
         Ok(())
     }
+    async fn append_quorum_proposal(
+        &mut self,
+        _proposal: &Proposal<SeqTypes, QuorumProposal<SeqTypes>>,
+    ) -> anyhow::Result<()> {
+        Ok(())
+    }
 }