VID share distribution algorithm (2) (#2163)

* original commit * second set of changes * increase test duration * Merge develop into rm/vid-share-distribution-algo-take2 * build, lint, etc examples by default * remove quorum membership from DA * remove commitment calculation from payload * remove quorum membership OTW * merge
EspressoSystems · Dec 8, 2023 · 81722f1 · 81722f1
1 parent efee8ff
commit 81722f1
Show file tree

Hide file tree

Showing 21 changed files with 176 additions and 100 deletions.
diff --git a/config/ValidatorConfigOutput b/config/ValidatorConfigOutput
@@ -1 +1 @@
-ValidatorConfig { public_key: BLSPubKey { pub_key: VerKey((QuadExtField(2264797523581107490935262917175769123227923636811928330606075281145117212394 + 15807017392833049888165434456991157794698032464874424842715555348468160607934 * u), QuadExtField(7996517616082121122160563552650547601395271017260499735456299700133762512689 + 7504045709281061282278228438613345070383424761478787301859187055302953740948 * u), QuadExtField(1515973040548822760825076242090160370742046237881440422068330135941139244581 + 20251846261653098602911417004145145971080304248810966341160788194007704966108 * u))) }, private_key: BLSPrivKey { priv_key: SignKey(BigInt([3505488234151006356, 6655477166151225138, 3291219027844407676, 2153641080015542578])) }, stake_value: 1, state_key_pair: StateKeyPair(KeyPair { sk: SignKey(BigInt([2822822805887490846, 6664316196088353173, 4926510007447087464, 116097479308258694])), vk: VerKey(Projective { x: BigInt([11315198235793138814, 4744451806709910489, 6921831025042192557, 1125393823825936625]), y: BigInt([13035879815613524256, 18225673961538637854, 12006860967936477969, 1516668567229692859]), t: BigInt([13450777528397789701, 12242009376162249168, 12596256366242272750, 3368076418495976469]), z: BigInt([10465708325245823445, 13967918689717629445, 14943426723808572731, 621075342718756551]) }) }) }
+ValidatorConfig { public_key: BLSPubKey { pub_key: VerKey((QuadExtField(2264797523581107490935262917175769123227923636811928330606075281145117212394 + 15807017392833049888165434456991157794698032464874424842715555348468160607934 * u), QuadExtField(7996517616082121122160563552650547601395271017260499735456299700133762512689 + 7504045709281061282278228438613345070383424761478787301859187055302953740948 * u), QuadExtField(1515973040548822760825076242090160370742046237881440422068330135941139244581 + 20251846261653098602911417004145145971080304248810966341160788194007704966108 * u))) }, private_key: BLSPrivKey { priv_key: SignKey(BigInt([3505488234151006356, 6655477166151225138, 3291219027844407676, 2153641080015542578])) }, stake_value: 1 }
diff --git a/crates/hotshot/Cargo.toml b/crates/hotshot/Cargo.toml
@@ -21,74 +21,60 @@ hotshot-testing = []
 # libp2p
 [[example]]
 name = "validator-libp2p"
-required-features = ["libp2p/rsa"]
 path = "examples/libp2p/validator.rs"
 
 [[example]]
 name = "multi-validator-libp2p"
-required-features = ["libp2p/rsa"]
 path = "examples/libp2p/multi-validator.rs"
 
 [[example]]
 name = "orchestrator-libp2p"
-required-features = ["libp2p/rsa"]
 path = "examples/libp2p/orchestrator.rs"
 
 [[example]]
 name = "all-libp2p"
-required-features = ["libp2p/rsa"]
 path = "examples/libp2p/all.rs"
 
 # webserver
 [[example]]
 name = "webserver"
-required-features = ["libp2p/rsa"]
 path = "examples/webserver/webserver.rs"
 
 [[example]]
 name = "orchestrator-webserver"
-required-features = ["libp2p/rsa"]
 path = "examples/webserver/orchestrator.rs"
 
 [[example]]
 name = "validator-webserver"
-required-features = ["libp2p/rsa"]
 path = "examples/webserver/validator.rs"
 
 [[example]]
 name = "multi-validator-webserver"
-required-features = ["libp2p/rsa"]
 path = "examples/webserver/multi-validator.rs"
 
 [[example]]
 name = "multi-webserver"
-required-features = ["libp2p/rsa"]
 path = "examples/webserver/multi-webserver.rs"
 
 [[example]]
 name = "all-webserver"
-required-features = ["libp2p/rsa"]
 path = "examples/webserver/all.rs"
 
 # combined
 [[example]]
 name = "all-combined"
-required-features = ["libp2p/rsa"]
 path = "examples/combined/all.rs"
 
 [[example]]
 name = "multi-validator-combined"
-required-features = ["libp2p/rsa"]
 path = "examples/combined/multi-validator.rs"
 
 [[example]]
 name = "validator-combined"
-required-features = ["libp2p/rsa"]
 path = "examples/combined/validator.rs"
 
 [[example]]
 name = "orchestrator-combined"
-required-features = ["libp2p/rsa"]
 path = "examples/combined/orchestrator.rs"
 
 [dependencies]

diff --git a/crates/hotshot/examples/combined/multi-validator.rs b/crates/hotshot/examples/combined/multi-validator.rs
@@ -41,7 +41,7 @@ async fn main() {
     tracing::error!("connecting to orchestrator at {:?}", args.url);
     let mut nodes = Vec::new();
     for _ in 0..args.num_nodes {
-        let url: String = args.url.clone();
+        let url = args.url.clone();
 
         let node = async_spawn(async move {
             infra::main_entry_point::<

diff --git a/crates/hotshot/examples/infra/mod.rs b/crates/hotshot/examples/infra/mod.rs
@@ -153,9 +153,9 @@ async fn webserver_network_from_config<TYPES: NodeType>(
     let WebServerConfig {
         url,
         wait_between_polls,
-    }: WebServerConfig = config.clone().web_server_config.unwrap();
+    }: WebServerConfig = config.web_server_config.unwrap();
 
-    WebServerNetwork::create(url, wait_between_polls, pub_key.clone(), false)
+    WebServerNetwork::create(url, wait_between_polls, pub_key, false)
 }
 
 async fn libp2p_network_from_config<TYPES: NodeType>(

diff --git a/crates/hotshot/examples/libp2p/multi-validator.rs b/crates/hotshot/examples/libp2p/multi-validator.rs
@@ -41,7 +41,7 @@ async fn main() {
     tracing::error!("connecting to orchestrator at {:?}", args.url);
     let mut nodes = Vec::new();
     for _ in 0..args.num_nodes {
-        let url: String = args.url.clone();
+        let url = args.url.clone();
 
         let node = async_spawn(async move {
             infra::main_entry_point::<

diff --git a/crates/hotshot/src/tasks/mod.rs b/crates/hotshot/src/tasks/mod.rs
@@ -23,6 +23,7 @@ use hotshot_task_impls::{
     vid::{VIDTaskState, VIDTaskTypes},
     view_sync::{ViewSyncTaskState, ViewSyncTaskStateTypes},
 };
+use hotshot_types::traits::election::Membership;
 use hotshot_types::{
     event::Event,
     message::Messages,
@@ -206,7 +207,15 @@ pub async fn add_consensus_task<TYPES: NodeType, I: NodeImplementation<TYPES>>(
     let registry = task_runner.registry.clone();
     let (payload, metadata) = <TYPES::BlockPayload as BlockPayload>::genesis();
     // Impossible for `unwrap` to fail on the genesis payload.
-    let payload_commitment = vid_commitment(&payload.encode().unwrap().collect());
+    let payload_commitment = vid_commitment(
+        &payload.encode().unwrap().collect(),
+        handle
+            .hotshot
+            .inner
+            .memberships
+            .quorum_membership
+            .total_nodes(),
+    );
     // build the consensus task
     let consensus_state = ConsensusTaskState {
         registry: registry.clone(),
@@ -353,6 +362,7 @@ pub async fn add_da_task<TYPES: NodeType, I: NodeImplementation<TYPES>>(
         consensus: handle.hotshot.get_consensus(),
         da_membership: c_api.inner.memberships.da_membership.clone().into(),
         da_network: c_api.inner.networks.da_network.clone().into(),
+        quorum_membership: c_api.inner.memberships.quorum_membership.clone().into(),
         cur_view: TYPES::Time::new(0),
         vote_collector: None,
         event_stream: event_stream.clone(),

diff --git a/crates/hotshot/src/traits/storage/memory_storage.rs b/crates/hotshot/src/traits/storage/memory_storage.rs
@@ -117,7 +117,10 @@ mod test {
     use hotshot_types::{
         data::{fake_commitment, genesis_proposer_id, Leaf},
         simple_certificate::QuorumCertificate,
-        traits::{node_implementation::NodeType, state::ConsensusTime},
+        traits::{
+            block_contents::genesis_vid_commitment, node_implementation::NodeType,
+            state::ConsensusTime,
+        },
     };
     use std::marker::PhantomData;
     use tracing::instrument;
@@ -126,7 +129,7 @@ mod test {
         let payload = TestBlockPayload::genesis();
         let header = TestBlockHeader {
             block_number: 0,
-            payload_commitment: payload.payload_commitment,
+            payload_commitment: genesis_vid_commitment(),
         };
         let dummy_leaf_commit = fake_commitment::<Leaf<TestTypes>>();
         let data = hotshot_types::simple_vote::QuorumData {

diff --git a/crates/task-impls/src/consensus.rs b/crates/task-impls/src/consensus.rs
@@ -618,12 +618,8 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, A: ConsensusApi<TYPES, I> +
                                         encoded_txns.clone().into_iter(),
                                         leaf.get_block_header().metadata(),
                                     );
-                                    if let Err(e) = leaf.fill_block_payload(payload) {
-                                        error!(
-                                            "Saved block payload and commitment don't match: {:?}",
-                                            e
-                                        );
-                                    }
+
+                                    leaf.fill_block_payload_unchecked(payload);
                                 }
 
                                 leaf_views.push(leaf.clone());

diff --git a/crates/task-impls/src/da.rs b/crates/task-impls/src/da.rs
@@ -57,6 +57,11 @@ pub struct DATaskState<
     /// Membership for the DA committee
     pub da_membership: Arc<TYPES::Membership>,
 
+    /// Membership for the quorum committee
+    /// We need this only for calculating the proper VID scheme
+    /// from the number of nodes in the quorum.
+    pub quorum_membership: Arc<TYPES::Membership>,
+
     /// Network for DA
     pub da_network: Arc<I::CommitteeNetwork>,
 
@@ -113,7 +118,10 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, A: ConsensusApi<TYPES, I> +
                     return None;
                 }
 
-                let payload_commitment = vid_commitment(&proposal.data.encoded_transactions);
+                let payload_commitment = vid_commitment(
+                    &proposal.data.encoded_transactions,
+                    self.quorum_membership.total_nodes(),
+                );
                 let encoded_transactions_hash = Sha256::digest(&proposal.data.encoded_transactions);
 
                 // ED Is this the right leader?

diff --git a/crates/task-impls/src/events.rs b/crates/task-impls/src/events.rs
@@ -92,7 +92,7 @@ pub enum HotShotEvent<TYPES: NodeType> {
         VidCommitment,
         <TYPES::BlockPayload as BlockPayload>::Metadata,
     ),
-    /// Event when the transactions task has sequenced transactions. Contains the encoded transactions
+    /// Event when the transactions task has sequenced transactions. Contains the encoded transactions, the metadata, and the view number
     TransactionsSequenced(
         Vec<u8>,
         <TYPES::BlockPayload as BlockPayload>::Metadata,

diff --git a/crates/task-impls/src/network.rs b/crates/task-impls/src/network.rs
@@ -56,7 +56,7 @@ impl<TYPES: NodeType> NetworkMessageTaskState<TYPES> {
                     let event = match consensus_message.0 {
                         Either::Left(general_message) => match general_message {
                             GeneralConsensusMessage::Proposal(proposal) => {
-                                HotShotEvent::QuorumProposalRecv(proposal.clone(), sender)
+                                HotShotEvent::QuorumProposalRecv(proposal, sender)
                             }
                             GeneralConsensusMessage::Vote(vote) => {
                                 HotShotEvent::QuorumVoteRecv(vote.clone())
@@ -92,7 +92,7 @@ impl<TYPES: NodeType> NetworkMessageTaskState<TYPES> {
                         },
                         Either::Right(committee_message) => match committee_message {
                             CommitteeConsensusMessage::DAProposal(proposal) => {
-                                HotShotEvent::DAProposalRecv(proposal.clone(), sender)
+                                HotShotEvent::DAProposalRecv(proposal, sender)
                             }
                             CommitteeConsensusMessage::DAVote(vote) => {
                                 HotShotEvent::DAVoteRecv(vote.clone())
@@ -236,7 +236,7 @@ impl<TYPES: NodeType, COMMCHANNEL: CommunicationChannel<TYPES>>
             HotShotEvent::ViewSyncPreCommitCertificate2Send(certificate, sender) => (
                 sender,
                 MessageKind::<TYPES>::from_consensus_message(SequencingMessage(Left(
-                    GeneralConsensusMessage::ViewSyncPreCommitCertificate(certificate.clone()),
+                    GeneralConsensusMessage::ViewSyncPreCommitCertificate(certificate),
                 ))),
                 TransmitType::Broadcast,
                 None,

diff --git a/crates/task-impls/src/vid.rs b/crates/task-impls/src/vid.rs
@@ -20,10 +20,7 @@ use hotshot_types::{
 };
 use hotshot_types::{
     data::{test_srs, VidScheme, VidSchemeTrait},
-    traits::{
-        block_contents::{NUM_CHUNKS, NUM_STORAGE_NODES},
-        network::ConsensusIntentEvent,
-    },
+    traits::network::ConsensusIntentEvent,
 };
 
 use hotshot_task::event_stream::EventStream;
@@ -54,7 +51,7 @@ pub struct VIDTaskState<
     pub consensus: Arc<RwLock<Consensus<TYPES>>>,
     /// Network for all nodes
     pub network: Arc<I::QuorumNetwork>,
-    /// Membership for teh quorum
+    /// Membership for the quorum
     pub membership: Arc<TYPES::Membership>,
     /// This Nodes Public Key
     pub public_key: TYPES::SignatureKey,
@@ -81,12 +78,19 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, A: ConsensusApi<TYPES, I> +
     ) -> Option<HotShotTaskCompleted> {
         match event {
             HotShotEvent::TransactionsSequenced(encoded_transactions, metadata, view_number) => {
+                // get the number of quorum committee members to be used for VID calculation
+                let num_quorum_committee = self.membership.total_nodes();
+
                 // TODO <https://github.com/EspressoSystems/HotShot/issues/1686>
-                let srs = test_srs(NUM_STORAGE_NODES);
-                // TODO We are using constant numbers for now, but they will change as the quorum size
-                // changes.
-                // TODO <https://github.com/EspressoSystems/HotShot/issues/1693>
-                let vid = VidScheme::new(NUM_CHUNKS, NUM_STORAGE_NODES, &srs).unwrap();
+                let srs = test_srs(num_quorum_committee);
+
+                // calculate the last power of two
+                // TODO change after https://github.com/EspressoSystems/jellyfish/issues/339
+                // issue: https://github.com/EspressoSystems/HotShot/issues/2152
+                let chunk_size = 1 << num_quorum_committee.ilog2();
+
+                // calculate vid shares
+                let vid = VidScheme::new(chunk_size, num_quorum_committee, &srs).unwrap();
                 let vid_disperse = vid.disperse(encoded_transactions.clone()).unwrap();
 
                 // send the commitment and metadata to consensus for block building
@@ -100,12 +104,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, A: ConsensusApi<TYPES, I> +
                 // send the block to the VID dispersal function
                 self.event_stream
                     .publish(HotShotEvent::BlockReady(
-                        VidDisperse {
-                            view_number,
-                            payload_commitment: vid_disperse.commit,
-                            shares: vid_disperse.shares,
-                            common: vid_disperse.common,
-                        },
+                        VidDisperse::from_membership(view_number, vid_disperse, &self.membership),
                         view_number,
                     ))
                     .await;

diff --git a/crates/testing/src/block_types.rs b/crates/testing/src/block_types.rs
@@ -5,9 +5,9 @@ use std::{
 
 use commit::{Commitment, Committable, RawCommitmentBuilder};
 use hotshot_types::{
-    data::{BlockError, VidCommitment, VidScheme, VidSchemeTrait},
+    data::{BlockError, VidCommitment},
     traits::{
-        block_contents::{vid_commitment, BlockHeader, Transaction},
+        block_contents::{genesis_vid_commitment, BlockHeader, Transaction},
         state::TestableBlock,
         BlockPayload,
     },
@@ -67,8 +67,6 @@ impl Transaction for TestTransaction {}
 pub struct TestBlockPayload {
     /// List of transactions.
     pub transactions: Vec<TestTransaction>,
-    /// VID commitment to the block payload.
-    pub payload_commitment: <VidScheme as VidSchemeTrait>::Commit,
 }
 
 impl TestBlockPayload {
@@ -80,10 +78,8 @@ impl TestBlockPayload {
     pub fn genesis() -> Self {
         let txns: Vec<u8> = vec![0];
         // It's impossible for `encode` to fail because the transaciton length is very small.
-        let encoded = TestTransaction::encode(vec![TestTransaction(txns.clone())]).unwrap();
         TestBlockPayload {
             transactions: vec![TestTransaction(txns)],
-            payload_commitment: vid_commitment(&encoded),
         }
     }
 }
@@ -114,11 +110,9 @@ impl BlockPayload for TestBlockPayload {
         transactions: impl IntoIterator<Item = Self::Transaction>,
     ) -> Result<(Self, Self::Metadata), Self::Error> {
         let txns_vec: Vec<TestTransaction> = transactions.into_iter().collect();
-        let encoded = TestTransaction::encode(txns_vec.clone())?;
         Ok((
             Self {
                 transactions: txns_vec,
-                payload_commitment: vid_commitment(&encoded),
             },
             (),
         ))
@@ -146,10 +140,7 @@ impl BlockPayload for TestBlockPayload {
             current_index = next_index;
         }
 
-        Self {
-            transactions,
-            payload_commitment: vid_commitment(&encoded_vec),
-        }
+        Self { transactions }
     }
 
     fn genesis() -> (Self, Self::Metadata) {
@@ -200,7 +191,7 @@ impl BlockHeader for TestBlockHeader {
         (
             Self {
                 block_number: 0,
-                payload_commitment: payload.payload_commitment,
+                payload_commitment: genesis_vid_commitment(),
             },
             payload,
             metadata,
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		ValidatorConfig { public_key: BLSPubKey { pub_key: VerKey((QuadExtField(2264797523581107490935262917175769123227923636811928330606075281145117212394 + 15807017392833049888165434456991157794698032464874424842715555348468160607934 * u), QuadExtField(7996517616082121122160563552650547601395271017260499735456299700133762512689 + 7504045709281061282278228438613345070383424761478787301859187055302953740948 * u), QuadExtField(1515973040548822760825076242090160370742046237881440422068330135941139244581 + 20251846261653098602911417004145145971080304248810966341160788194007704966108 * u))) }, private_key: BLSPrivKey { priv_key: SignKey(BigInt([3505488234151006356, 6655477166151225138, 3291219027844407676, 2153641080015542578])) }, stake_value: 1, state_key_pair: StateKeyPair(KeyPair { sk: SignKey(BigInt([2822822805887490846, 6664316196088353173, 4926510007447087464, 116097479308258694])), vk: VerKey(Projective { x: BigInt([11315198235793138814, 4744451806709910489, 6921831025042192557, 1125393823825936625]), y: BigInt([13035879815613524256, 18225673961538637854, 12006860967936477969, 1516668567229692859]), t: BigInt([13450777528397789701, 12242009376162249168, 12596256366242272750, 3368076418495976469]), z: BigInt([10465708325245823445, 13967918689717629445, 14943426723808572731, 621075342718756551]) }) }) }
		ValidatorConfig { public_key: BLSPubKey { pub_key: VerKey((QuadExtField(2264797523581107490935262917175769123227923636811928330606075281145117212394 + 15807017392833049888165434456991157794698032464874424842715555348468160607934 * u), QuadExtField(7996517616082121122160563552650547601395271017260499735456299700133762512689 + 7504045709281061282278228438613345070383424761478787301859187055302953740948 * u), QuadExtField(1515973040548822760825076242090160370742046237881440422068330135941139244581 + 20251846261653098602911417004145145971080304248810966341160788194007704966108 * u))) }, private_key: BLSPrivKey { priv_key: SignKey(BigInt([3505488234151006356, 6655477166151225138, 3291219027844407676, 2153641080015542578])) }, stake_value: 1 }