From 53f551b4f7b867fb518a798b64ea3d1c288c26f7 Mon Sep 17 00:00:00 2001 From: Bowen Date: Fri, 3 Jan 2025 00:11:58 -0800 Subject: [PATCH 01/11] move codec logic into one file --- .gitignore | 1 + bin/client/justfile | 3 + bin/host/src/eigenda_fetcher/mod.rs | 34 ++++------ crates/eigenda/src/codec.rs | 98 +++++++++++++++++++++++++++++ crates/eigenda/src/eigenda_blobs.rs | 4 +- crates/eigenda/src/eigenda_data.rs | 27 +------- crates/eigenda/src/lib.rs | 4 ++ 7 files changed, 123 insertions(+), 48 deletions(-) create mode 100644 crates/eigenda/src/codec.rs diff --git a/.gitignore b/.gitignore index b9dd3f1..bf8d9b1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ target/ data/ optimism/ +tags diff --git a/bin/client/justfile b/bin/client/justfile index 7aabde9..b682b95 100644 --- a/bin/client/justfile +++ b/bin/client/justfile @@ -119,6 +119,9 @@ run-client-native block_number l1_rpc l1_beacon_rpc l2_rpc rollup_node_rpc rollu # Move to the workspace root cd $(git rev-parse --show-toplevel) + rm -rf ./data + mkdir ./data + echo "Running host program with native client program..." cargo r --bin hokulea-host -- \ --l1-head $L1_HEAD \ diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 95e4810..bbfb093 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -7,7 +7,8 @@ use alloy_provider::ReqwestProvider; use alloy_rlp::Decodable; use anyhow::{anyhow, Result}; use core::panic; -use hokulea_eigenda::BlobInfo; +use hokulea_eigenda::{BlobInfo}; +use hokulea_eigenda::encode_eigenda_blob; use hokulea_eigenda::BLOB_ENCODING_VERSION_0; use hokulea_proof::hint::{ExtendedHint, ExtendedHintType}; use kona_host::{blobs::OnlineBlobProvider, fetcher::Fetcher, kv::KeyValueStore}; @@ -141,45 +142,34 @@ where if hint_type == ExtendedHintType::EigenDACommitment { let cert = hint_data; - info!(target: "fetcher_with_eigenda_support", "Fetching eigenda commitment cert: {:?}", cert); + trace!(target: "fetcher_with_eigenda_support", "Fetching eigenda commitment cert: {:?}", cert); // Fetch the blob sidecar from the blob provider. let rollup_data = self .eigenda_blob_provider .fetch_eigenda_blob(&cert) .await .map_err(|e| anyhow!("Failed to fetch eigenda blob: {e}"))?; - // Acquire a lock on the key-value store and set the preimages. let mut kv_write_lock = self.kv_store.write().await; + // the fourth because 0x01010000 in the beginning is metadata - let rollup_data_len = rollup_data.len() as u32; let item_slice = cert.as_ref(); let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..]).unwrap(); - // Todo ensure data_length is always power of 2. Proxy made mistake + // TODO ensure data_length is always power of 2. Proxy made mistake let data_size = cert_blob_info.blob_header.data_length as u64; let blob_length: u64 = data_size / 32; - // encode to become raw blob - let codec_rollup_data = helpers::convert_by_padding_empty_byte(rollup_data.as_ref()); - let codec_rollup_data_len = codec_rollup_data.len() as u32; - - let mut raw_blob = vec![0u8; data_size as usize]; + let raw_blob = encode_eigenda_blob(rollup_data.as_ref()); + trace!(target: "fetcher_with_eigenda_support", "Fetching ssize size: {:?} {}", raw_blob.len() , data_size); - if 32 + codec_rollup_data_len as u64 > data_size { - return Err(anyhow!("data size is less than reconstructed data codec_rollup_data_len {} data_size {}", codec_rollup_data_len, data_size)); + if raw_blob.len() != data_size as usize { + return Err( + anyhow!("data size from cert does not equal to reconstructed data codec_rollup_data_len {} data_size {}", + raw_blob.len(), data_size)); } - // blob header - // https://github.com/Layr-Labs/eigenda/blob/f8b0d31d65b29e60172507074922668f4ca89420/api/clients/codecs/default_blob_codec.go#L25 - // raw blob the immediate data just before taking IFFT - raw_blob[1] = BLOB_ENCODING_VERSION_0; - raw_blob[2..6].copy_from_slice(&rollup_data_len.to_be_bytes()); - - // encode length as uint32 - raw_blob[32..(32 + codec_rollup_data_len as usize)].copy_from_slice(&codec_rollup_data); - // Write all the field elements to the key-value store. // The preimage oracle key for each field element is the keccak256 hash of // `abi.encodePacked(cert.KZGCommitment, uint256(i))` @@ -189,7 +179,7 @@ where blob_key[..32].copy_from_slice(cert_blob_info.blob_header.commitment.x.as_ref()); blob_key[32..64].copy_from_slice(cert_blob_info.blob_header.commitment.y.as_ref()); - info!("cert_blob_info blob_length {:?}", blob_length); + trace!("cert_blob_info blob_length {:?}", blob_length); for i in 0..blob_length { blob_key[88..].copy_from_slice(i.to_be_bytes().as_ref()); diff --git a/crates/eigenda/src/codec.rs b/crates/eigenda/src/codec.rs new file mode 100644 index 0000000..c886d40 --- /dev/null +++ b/crates/eigenda/src/codec.rs @@ -0,0 +1,98 @@ +use alloy_primitives::Bytes; +use bytes::buf::Buf; +use rust_kzg_bn254::helpers; +use crate::BLOB_ENCODING_VERSION_0; +use kona_derive::errors::BlobDecodingError; +use alloc::vec::Vec; + +/// encoded data into an eigenda blob. The output is always power of 2 +pub fn encode_eigenda_blob(rollup_data: &[u8]) -> Bytes { + let rollup_data_size = rollup_data.len() as u32; + + // encode to become raw blob + let codec_rollup_data = helpers::convert_by_padding_empty_byte(rollup_data.as_ref()); + + let blob_payload_size = codec_rollup_data.len(); + + let blob_size = blob_payload_size + 32; + let blob_size = blob_size.next_power_of_two(); + + let mut raw_blob = Vec::::with_capacity(blob_size as usize); + for i in 0..blob_size { + raw_blob.push(0); + } + + raw_blob[1] = BLOB_ENCODING_VERSION_0; + raw_blob[2..6].copy_from_slice(&rollup_data_size.to_be_bytes()); + + // encode length as uint32 + raw_blob[32..(32 + blob_payload_size as usize)].copy_from_slice(&codec_rollup_data); + + Bytes::from(raw_blob) +} + + +/// decode data into an eigenda blob +pub fn decode_eigenda_blob(blob: &Bytes) -> Result { + if blob.len() < 32 { + return Err(BlobDecodingError::InvalidLength); + } + + info!(target: "eigenda-datasource", "padded_eigenda_blob {:?}", blob); + + // see https://github.com/Layr-Labs/eigenda/blob/f8b0d31d65b29e60172507074922668f4ca89420/api/clients/codecs/default_blob_codec.go#L44 + let content_size = blob.slice(2..6).get_u32(); + info!(target: "eigenda-datasource", "content_size {:?}", content_size); + + // the first 32 Bytes are reserved as the header field element + let codec_data = blob.slice(32..); + + // rust kzg bn254 impl already + let blob_content = + helpers::remove_empty_byte_from_padded_bytes_unchecked(codec_data.as_ref()); + let blob_content: Bytes = blob_content.into(); + + if blob_content.len() < content_size as usize { + return Err(BlobDecodingError::InvalidLength); + } + Ok(blob_content.slice(..content_size as usize)) +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::vec; + use alloy_primitives::Bytes; + use kona_derive::errors::BlobDecodingError; + + #[test] + fn test_decode_success() { + let content = vec![1, 2, 3, 4]; + let data = encode_eigenda_blob(&content); + let data_len = data.len(); + assert!(data_len.is_power_of_two()); + + let result = decode_eigenda_blob(&data); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), Bytes::from(content)); + } + + #[test] + fn test_decode_success_empty() { + let content = vec![]; + let data = encode_eigenda_blob(&content); + let result = decode_eigenda_blob(&data); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), Bytes::from(content)); + } + + #[test] + fn test_decode_error_invalid_length() { + let content = vec![1, 2, 3, 4]; + let mut data = encode_eigenda_blob(&content); + data.truncate(33); + let result = decode_eigenda_blob(&data); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), BlobDecodingError::InvalidLength); + } +} diff --git a/crates/eigenda/src/eigenda_blobs.rs b/crates/eigenda/src/eigenda_blobs.rs index 741176c..c3c435d 100644 --- a/crates/eigenda/src/eigenda_blobs.rs +++ b/crates/eigenda/src/eigenda_blobs.rs @@ -48,8 +48,8 @@ where // Otherwise, ignore blob and recurse next. match next_data.decode() { Ok(d) => Ok(d), - Err(_) => { - warn!(target: "blob-source", "Failed to decode blob data, skipping"); + Err(e) => { + warn!(target: "blob-source", "Failed to decode blob data, skipping {}", e); panic!() } } diff --git a/crates/eigenda/src/eigenda_data.rs b/crates/eigenda/src/eigenda_data.rs index d525fa1..0574934 100644 --- a/crates/eigenda/src/eigenda_data.rs +++ b/crates/eigenda/src/eigenda_data.rs @@ -1,6 +1,5 @@ +use crate::codec; use alloy_primitives::Bytes; -use bytes::buf::Buf; - use kona_derive::errors::BlobDecodingError; use rust_kzg_bn254::helpers; @@ -16,31 +15,11 @@ impl EigenDABlobData { /// Decodes the blob into raw byte data. /// Returns a [BlobDecodingError] if the blob is invalid. pub(crate) fn decode(&self) -> Result { - if self.blob.len() < 32 { - return Err(BlobDecodingError::InvalidLength); - } - - info!(target: "eigenda-datasource", "padded_eigenda_blob {:?}", self.blob); - - // see https://github.com/Layr-Labs/eigenda/blob/f8b0d31d65b29e60172507074922668f4ca89420/api/clients/codecs/default_blob_codec.go#L44 - let content_size = self.blob.slice(2..6).get_u32(); - info!(target: "eigenda-datasource", "content_size {:?}", content_size); - - // the first 32 Bytes are reserved as the header field element - let codec_data = self.blob.slice(32..); - - // rust kzg bn254 impl already - let blob_content = - helpers::remove_empty_byte_from_padded_bytes_unchecked(codec_data.as_ref()); - let blob_content: Bytes = blob_content.into(); - - if blob_content.len() < content_size as usize { - return Err(BlobDecodingError::InvalidLength); - } + let rollup_blob = codec::decode_eigenda_blob(&self.blob)?; // might insert a FFT here, // take data - Ok(blob_content.slice(..content_size as usize)) + Ok(rollup_blob) } } diff --git a/crates/eigenda/src/lib.rs b/crates/eigenda/src/lib.rs index 1d02e90..1fb39b2 100644 --- a/crates/eigenda/src/lib.rs +++ b/crates/eigenda/src/lib.rs @@ -30,6 +30,10 @@ pub use eigenda_data::EigenDABlobData; mod certificate; pub use certificate::BlobInfo; +pub mod codec; +pub use codec::decode_eigenda_blob; +pub use codec::encode_eigenda_blob; + mod constant; pub use constant::BLOB_ENCODING_VERSION_0; pub use constant::STALE_GAP; From 5bb22cca1c0f41e8684dbe7f9af319586133ed5c Mon Sep 17 00:00:00 2001 From: Bowen Date: Fri, 3 Jan 2025 00:12:16 -0800 Subject: [PATCH 02/11] fix lint --- bin/host/src/eigenda_fetcher/mod.rs | 3 +-- crates/eigenda/src/codec.rs | 10 ++++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index bbfb093..8001831 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -7,8 +7,8 @@ use alloy_provider::ReqwestProvider; use alloy_rlp::Decodable; use anyhow::{anyhow, Result}; use core::panic; -use hokulea_eigenda::{BlobInfo}; use hokulea_eigenda::encode_eigenda_blob; +use hokulea_eigenda::BlobInfo; use hokulea_eigenda::BLOB_ENCODING_VERSION_0; use hokulea_proof::hint::{ExtendedHint, ExtendedHintType}; use kona_host::{blobs::OnlineBlobProvider, fetcher::Fetcher, kv::KeyValueStore}; @@ -152,7 +152,6 @@ where // Acquire a lock on the key-value store and set the preimages. let mut kv_write_lock = self.kv_store.write().await; - // the fourth because 0x01010000 in the beginning is metadata let item_slice = cert.as_ref(); let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..]).unwrap(); diff --git a/crates/eigenda/src/codec.rs b/crates/eigenda/src/codec.rs index c886d40..bee8937 100644 --- a/crates/eigenda/src/codec.rs +++ b/crates/eigenda/src/codec.rs @@ -1,9 +1,9 @@ +use crate::BLOB_ENCODING_VERSION_0; +use alloc::vec::Vec; use alloy_primitives::Bytes; use bytes::buf::Buf; -use rust_kzg_bn254::helpers; -use crate::BLOB_ENCODING_VERSION_0; use kona_derive::errors::BlobDecodingError; -use alloc::vec::Vec; +use rust_kzg_bn254::helpers; /// encoded data into an eigenda blob. The output is always power of 2 pub fn encode_eigenda_blob(rollup_data: &[u8]) -> Bytes { @@ -31,7 +31,6 @@ pub fn encode_eigenda_blob(rollup_data: &[u8]) -> Bytes { Bytes::from(raw_blob) } - /// decode data into an eigenda blob pub fn decode_eigenda_blob(blob: &Bytes) -> Result { if blob.len() < 32 { @@ -48,8 +47,7 @@ pub fn decode_eigenda_blob(blob: &Bytes) -> Result { let codec_data = blob.slice(32..); // rust kzg bn254 impl already - let blob_content = - helpers::remove_empty_byte_from_padded_bytes_unchecked(codec_data.as_ref()); + let blob_content = helpers::remove_empty_byte_from_padded_bytes_unchecked(codec_data.as_ref()); let blob_content: Bytes = blob_content.into(); if blob_content.len() < content_size as usize { From 85e61bdaf14b87dfd2fa17c692e9e4c6a8b7864d Mon Sep 17 00:00:00 2001 From: Bowen Date: Fri, 3 Jan 2025 10:35:17 -0800 Subject: [PATCH 03/11] remove codec and add comments, make blob from EigendaBlobDAta public --- bin/host/src/eigenda_fetcher/mod.rs | 2 + crates/eigenda/src/codec.rs | 96 ------------------------- crates/eigenda/src/eigenda_data.rs | 108 +++++++++++++++++++++++++--- crates/eigenda/src/lib.rs | 4 -- 4 files changed, 102 insertions(+), 108 deletions(-) delete mode 100644 crates/eigenda/src/codec.rs diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 8001831..5d5e02c 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -157,6 +157,8 @@ where let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..]).unwrap(); // TODO ensure data_length is always power of 2. Proxy made mistake + // Proxy should return a cert whose data_length measured in symbol (i.e. 32 Bytes) + // Currently, it returns number of bytes. We need to fix proxy and here later. let data_size = cert_blob_info.blob_header.data_length as u64; let blob_length: u64 = data_size / 32; diff --git a/crates/eigenda/src/codec.rs b/crates/eigenda/src/codec.rs deleted file mode 100644 index bee8937..0000000 --- a/crates/eigenda/src/codec.rs +++ /dev/null @@ -1,96 +0,0 @@ -use crate::BLOB_ENCODING_VERSION_0; -use alloc::vec::Vec; -use alloy_primitives::Bytes; -use bytes::buf::Buf; -use kona_derive::errors::BlobDecodingError; -use rust_kzg_bn254::helpers; - -/// encoded data into an eigenda blob. The output is always power of 2 -pub fn encode_eigenda_blob(rollup_data: &[u8]) -> Bytes { - let rollup_data_size = rollup_data.len() as u32; - - // encode to become raw blob - let codec_rollup_data = helpers::convert_by_padding_empty_byte(rollup_data.as_ref()); - - let blob_payload_size = codec_rollup_data.len(); - - let blob_size = blob_payload_size + 32; - let blob_size = blob_size.next_power_of_two(); - - let mut raw_blob = Vec::::with_capacity(blob_size as usize); - for i in 0..blob_size { - raw_blob.push(0); - } - - raw_blob[1] = BLOB_ENCODING_VERSION_0; - raw_blob[2..6].copy_from_slice(&rollup_data_size.to_be_bytes()); - - // encode length as uint32 - raw_blob[32..(32 + blob_payload_size as usize)].copy_from_slice(&codec_rollup_data); - - Bytes::from(raw_blob) -} - -/// decode data into an eigenda blob -pub fn decode_eigenda_blob(blob: &Bytes) -> Result { - if blob.len() < 32 { - return Err(BlobDecodingError::InvalidLength); - } - - info!(target: "eigenda-datasource", "padded_eigenda_blob {:?}", blob); - - // see https://github.com/Layr-Labs/eigenda/blob/f8b0d31d65b29e60172507074922668f4ca89420/api/clients/codecs/default_blob_codec.go#L44 - let content_size = blob.slice(2..6).get_u32(); - info!(target: "eigenda-datasource", "content_size {:?}", content_size); - - // the first 32 Bytes are reserved as the header field element - let codec_data = blob.slice(32..); - - // rust kzg bn254 impl already - let blob_content = helpers::remove_empty_byte_from_padded_bytes_unchecked(codec_data.as_ref()); - let blob_content: Bytes = blob_content.into(); - - if blob_content.len() < content_size as usize { - return Err(BlobDecodingError::InvalidLength); - } - Ok(blob_content.slice(..content_size as usize)) -} - -#[cfg(test)] -mod tests { - use super::*; - use alloc::vec; - use alloy_primitives::Bytes; - use kona_derive::errors::BlobDecodingError; - - #[test] - fn test_decode_success() { - let content = vec![1, 2, 3, 4]; - let data = encode_eigenda_blob(&content); - let data_len = data.len(); - assert!(data_len.is_power_of_two()); - - let result = decode_eigenda_blob(&data); - assert!(result.is_ok()); - assert_eq!(result.unwrap(), Bytes::from(content)); - } - - #[test] - fn test_decode_success_empty() { - let content = vec![]; - let data = encode_eigenda_blob(&content); - let result = decode_eigenda_blob(&data); - assert!(result.is_ok()); - assert_eq!(result.unwrap(), Bytes::from(content)); - } - - #[test] - fn test_decode_error_invalid_length() { - let content = vec![1, 2, 3, 4]; - let mut data = encode_eigenda_blob(&content); - data.truncate(33); - let result = decode_eigenda_blob(&data); - assert!(result.is_err()); - assert_eq!(result.unwrap_err(), BlobDecodingError::InvalidLength); - } -} diff --git a/crates/eigenda/src/eigenda_data.rs b/crates/eigenda/src/eigenda_data.rs index 0574934..73ef422 100644 --- a/crates/eigenda/src/eigenda_data.rs +++ b/crates/eigenda/src/eigenda_data.rs @@ -1,7 +1,8 @@ -use crate::codec; +use crate::BLOB_ENCODING_VERSION_0; +use alloc::vec; use alloy_primitives::Bytes; +use bytes::buf::Buf; use kona_derive::errors::BlobDecodingError; - use rust_kzg_bn254::helpers; #[derive(Default, Clone, Debug)] @@ -12,14 +13,70 @@ pub struct EigenDABlobData { } impl EigenDABlobData { - /// Decodes the blob into raw byte data. + /// Decodes the blob into raw byte data. Reverse of the encode function below /// Returns a [BlobDecodingError] if the blob is invalid. - pub(crate) fn decode(&self) -> Result { - let rollup_blob = codec::decode_eigenda_blob(&self.blob)?; - // might insert a FFT here, + pub fn decode(&self) -> Result { + let blob = &self.blob; + if blob.len() < 32 { + return Err(BlobDecodingError::InvalidLength); + } + + info!(target: "eigenda-datasource", "padded_eigenda_blob {:?}", blob); + + // see https://github.com/Layr-Labs/eigenda/blob/f8b0d31d65b29e60172507074922668f4ca89420/api/clients/codecs/default_blob_codec.go#L44 + let content_size = blob.slice(2..6).get_u32(); + info!(target: "eigenda-datasource", "content_size {:?}", content_size); + + // the first 32 Bytes are reserved as the header field element + let codec_data = blob.slice(32..); + + // rust kzg bn254 impl already + let blob_content = + helpers::remove_empty_byte_from_padded_bytes_unchecked(codec_data.as_ref()); + let blob_content: Bytes = blob_content.into(); + + if blob_content.len() < content_size as usize { + return Err(BlobDecodingError::InvalidLength); + } + Ok(blob_content.slice(..content_size as usize)) + } + + /// The encode function accepts an input of opaque rollup data array into an EigenDABlobData. + /// EigenDABlobData contains a header of 32 bytes and a transformation of input data + /// The 0 index byte of header is always 0, to comply to bn254 field element constraint + /// The 1 index byte of header is proxy encoding version. + /// The 2-4 indices of header are storing the length of the input rollup data in big endien + /// The payload is prepared by padding an empty byte for every 31 bytes from the rollup data + /// This matches exactly the eigenda proxy implementation, whose logic is in + /// https://github.com/Layr-Labs/eigenda/blob/master/encoding/utils/codec/codec.go#L12 + /// + /// The length of (header + payload) by the encode function is always power of 2 + /// The eigenda proxy does not take such constraint. + /// TODO it is possible to remove such power of 2 constraint, such that the client is not + /// relying on the data_length from eigenda cert. It might save some comm rounds between + /// host and client. + pub fn encode(rollup_data: &[u8]) -> Self { + let rollup_data_size = rollup_data.len() as u32; + + // encode to become raw blob + let codec_rollup_data = helpers::convert_by_padding_empty_byte(rollup_data.as_ref()); + + let blob_payload_size = codec_rollup_data.len(); + + let blob_size = blob_payload_size + 32; + let blob_size = blob_size.next_power_of_two(); + + let mut raw_blob = vec![0u8; blob_size as usize]; + + raw_blob[1] = BLOB_ENCODING_VERSION_0; + raw_blob[2..6].copy_from_slice(&rollup_data_size.to_be_bytes()); + + // encode length as uint32 + raw_blob[32..(32 + blob_payload_size as usize)].copy_from_slice(&codec_rollup_data); - // take data - Ok(rollup_blob) + Self { + blob: Bytes::from(raw_blob), + } } } @@ -69,4 +126,39 @@ mod tests { assert!(result.is_err()); assert_eq!(result.unwrap_err(), BlobDecodingError::InvalidLength); } + + #[test] + fn test_encode_and_decode_success() { + let rollup_data = vec![1, 2, 3, 4]; + let eigenda_blob = EigenDABlobData::encode(&rollup_data); + let data_len = eigenda_blob.blob.len(); + assert!(data_len.is_power_of_two()); + + let result = eigenda_blob.decode(); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), Bytes::from(rollup_data)); + } + + #[test] + fn test_encode_and_decode_success_empty() { + let rollup_data = vec![]; + let eigenda_blob = EigenDABlobData::encode(&rollup_data); + let data_len = eigenda_blob.blob.len(); + // 32 is eigenda blob header size + assert!(data_len == 32); + + let result = eigenda_blob.decode(); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), Bytes::from(rollup_data)); + } + + #[test] + fn test_encode_and_decode_error_invalid_length() { + let rollup_data = vec![1, 2, 3, 4]; + let mut eigenda_blob = EigenDABlobData::encode(&rollup_data); + eigenda_blob.blob.truncate(33); + let result = eigenda_blob.decode(); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), BlobDecodingError::InvalidLength); + } } diff --git a/crates/eigenda/src/lib.rs b/crates/eigenda/src/lib.rs index 1fb39b2..1d02e90 100644 --- a/crates/eigenda/src/lib.rs +++ b/crates/eigenda/src/lib.rs @@ -30,10 +30,6 @@ pub use eigenda_data::EigenDABlobData; mod certificate; pub use certificate::BlobInfo; -pub mod codec; -pub use codec::decode_eigenda_blob; -pub use codec::encode_eigenda_blob; - mod constant; pub use constant::BLOB_ENCODING_VERSION_0; pub use constant::STALE_GAP; From c476f1dfcfa4f3cc6f9856c91c59ee91d87c19f9 Mon Sep 17 00:00:00 2001 From: Bowen Date: Fri, 3 Jan 2025 10:36:31 -0800 Subject: [PATCH 04/11] fix lint --- bin/host/src/eigenda_fetcher/mod.rs | 11 +++++------ crates/eigenda/src/eigenda_data.rs | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 5d5e02c..9334dc9 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -7,8 +7,8 @@ use alloy_provider::ReqwestProvider; use alloy_rlp::Decodable; use anyhow::{anyhow, Result}; use core::panic; -use hokulea_eigenda::encode_eigenda_blob; use hokulea_eigenda::BlobInfo; +use hokulea_eigenda::EigenDABlobData; use hokulea_eigenda::BLOB_ENCODING_VERSION_0; use hokulea_proof::hint::{ExtendedHint, ExtendedHintType}; use kona_host::{blobs::OnlineBlobProvider, fetcher::Fetcher, kv::KeyValueStore}; @@ -162,13 +162,12 @@ where let data_size = cert_blob_info.blob_header.data_length as u64; let blob_length: u64 = data_size / 32; - let raw_blob = encode_eigenda_blob(rollup_data.as_ref()); - trace!(target: "fetcher_with_eigenda_support", "Fetching ssize size: {:?} {}", raw_blob.len() , data_size); + let eigenda_blob = EigenDABlobData::encode(rollup_data.as_ref()); - if raw_blob.len() != data_size as usize { + if eigenda_blob.blob.len() != data_size as usize { return Err( anyhow!("data size from cert does not equal to reconstructed data codec_rollup_data_len {} data_size {}", - raw_blob.len(), data_size)); + eigenda_blob.blob.len(), data_size)); } // Write all the field elements to the key-value store. @@ -192,7 +191,7 @@ where )?; kv_write_lock.set( PreimageKey::new(*blob_key_hash, PreimageKeyType::GlobalGeneric).into(), - raw_blob[(i as usize) << 5..(i as usize + 1) << 5].to_vec(), + eigenda_blob.blob[(i as usize) << 5..(i as usize + 1) << 5].to_vec(), )?; } diff --git a/crates/eigenda/src/eigenda_data.rs b/crates/eigenda/src/eigenda_data.rs index 73ef422..1133fc7 100644 --- a/crates/eigenda/src/eigenda_data.rs +++ b/crates/eigenda/src/eigenda_data.rs @@ -9,7 +9,7 @@ use rust_kzg_bn254::helpers; /// Represents the data structure for EigenDA Blob. pub struct EigenDABlobData { /// The calldata - pub(crate) blob: Bytes, + pub blob: Bytes, } impl EigenDABlobData { From 5b05e9dd5d9fede67405cd6e1e5c41dd1e8d6886 Mon Sep 17 00:00:00 2001 From: Bowen Date: Fri, 3 Jan 2025 10:49:02 -0800 Subject: [PATCH 05/11] fix doc --- crates/eigenda/src/eigenda_data.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/eigenda/src/eigenda_data.rs b/crates/eigenda/src/eigenda_data.rs index 1133fc7..cbd0b98 100644 --- a/crates/eigenda/src/eigenda_data.rs +++ b/crates/eigenda/src/eigenda_data.rs @@ -48,7 +48,7 @@ impl EigenDABlobData { /// The 2-4 indices of header are storing the length of the input rollup data in big endien /// The payload is prepared by padding an empty byte for every 31 bytes from the rollup data /// This matches exactly the eigenda proxy implementation, whose logic is in - /// https://github.com/Layr-Labs/eigenda/blob/master/encoding/utils/codec/codec.go#L12 + /// /// /// The length of (header + payload) by the encode function is always power of 2 /// The eigenda proxy does not take such constraint. From 52a7fd0902ce891c3525443daaff947a43a15009 Mon Sep 17 00:00:00 2001 From: Bowen Date: Fri, 3 Jan 2025 13:52:11 -0800 Subject: [PATCH 06/11] fix comments, fix proxy bug --- bin/host/src/eigenda_fetcher/mod.rs | 18 ++++++++++-------- crates/eigenda/src/constant.rs | 2 ++ crates/eigenda/src/eigenda_data.rs | 2 +- crates/eigenda/src/lib.rs | 1 + crates/proof/src/eigenda_provider.rs | 12 ++++++------ op-devnet.docker-compose.yml.patch | 2 +- 6 files changed, 21 insertions(+), 16 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 9334dc9..14bbfe2 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -9,14 +9,13 @@ use anyhow::{anyhow, Result}; use core::panic; use hokulea_eigenda::BlobInfo; use hokulea_eigenda::EigenDABlobData; -use hokulea_eigenda::BLOB_ENCODING_VERSION_0; +use hokulea_eigenda::BYTES_PER_FIELD_ELEMENT; use hokulea_proof::hint::{ExtendedHint, ExtendedHintType}; use kona_host::{blobs::OnlineBlobProvider, fetcher::Fetcher, kv::KeyValueStore}; use kona_preimage::{PreimageKey, PreimageKeyType}; -use rust_kzg_bn254::helpers; use std::sync::Arc; use tokio::sync::RwLock; -use tracing::{error, info, trace, warn}; +use tracing::{error, trace, warn}; /// The [FetcherWithEigenDASupport] struct wraps and extends kona's [Fetcher] struct with the ability /// to fetch preimages from EigenDA. @@ -159,15 +158,18 @@ where // TODO ensure data_length is always power of 2. Proxy made mistake // Proxy should return a cert whose data_length measured in symbol (i.e. 32 Bytes) // Currently, it returns number of bytes. We need to fix proxy and here later. - let data_size = cert_blob_info.blob_header.data_length as u64; - let blob_length: u64 = data_size / 32; + //let data_size = cert_blob_info.blob_header.data_length as u64; + //let blob_length: u64 = data_size / 32; + + let blob_length = cert_blob_info.blob_header.data_length as u64; + warn!("blob length: {:?}", blob_length); let eigenda_blob = EigenDABlobData::encode(rollup_data.as_ref()); - if eigenda_blob.blob.len() != data_size as usize { + if eigenda_blob.blob.len() != blob_length as usize * BYTES_PER_FIELD_ELEMENT { return Err( - anyhow!("data size from cert does not equal to reconstructed data codec_rollup_data_len {} data_size {}", - eigenda_blob.blob.len(), data_size)); + anyhow!("data size from cert does not equal to reconstructed data codec_rollup_data_len {} blob size {}", + eigenda_blob.blob.len(), blob_length as usize * BYTES_PER_FIELD_ELEMENT)); } // Write all the field elements to the key-value store. diff --git a/crates/eigenda/src/constant.rs b/crates/eigenda/src/constant.rs index 8a5a5c6..f07d4ac 100644 --- a/crates/eigenda/src/constant.rs +++ b/crates/eigenda/src/constant.rs @@ -3,3 +3,5 @@ pub const BLOB_ENCODING_VERSION_0: u8 = 0x0; /// TODO: make it part of rollup config pub const STALE_GAP: u64 = 100; +/// Number of fields for field element on bn254 +pub const BYTES_PER_FIELD_ELEMENT: usize = 32; diff --git a/crates/eigenda/src/eigenda_data.rs b/crates/eigenda/src/eigenda_data.rs index cbd0b98..a7cdffe 100644 --- a/crates/eigenda/src/eigenda_data.rs +++ b/crates/eigenda/src/eigenda_data.rs @@ -59,7 +59,7 @@ impl EigenDABlobData { let rollup_data_size = rollup_data.len() as u32; // encode to become raw blob - let codec_rollup_data = helpers::convert_by_padding_empty_byte(rollup_data.as_ref()); + let codec_rollup_data = helpers::convert_by_padding_empty_byte(rollup_data); let blob_payload_size = codec_rollup_data.len(); diff --git a/crates/eigenda/src/lib.rs b/crates/eigenda/src/lib.rs index 1d02e90..9219623 100644 --- a/crates/eigenda/src/lib.rs +++ b/crates/eigenda/src/lib.rs @@ -32,4 +32,5 @@ pub use certificate::BlobInfo; mod constant; pub use constant::BLOB_ENCODING_VERSION_0; +pub use constant::BYTES_PER_FIELD_ELEMENT; pub use constant::STALE_GAP; diff --git a/crates/proof/src/eigenda_provider.rs b/crates/proof/src/eigenda_provider.rs index e94fff6..5c62f23 100644 --- a/crates/proof/src/eigenda_provider.rs +++ b/crates/proof/src/eigenda_provider.rs @@ -2,7 +2,7 @@ use alloc::boxed::Box; use alloc::sync::Arc; use alloy_primitives::{keccak256, Bytes}; use async_trait::async_trait; -use hokulea_eigenda::{BlobInfo, EigenDABlobProvider}; +use hokulea_eigenda::{BlobInfo, EigenDABlobProvider, BYTES_PER_FIELD_ELEMENT}; use kona_preimage::{errors::PreimageOracleError, CommsClient, PreimageKey, PreimageKeyType}; use kona_proof::errors::OracleProviderError; @@ -51,7 +51,9 @@ impl EigenDABlobProvider for OracleEigenDAProvider let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..]).unwrap(); info!("cert_blob_info {:?}", cert_blob_info); - let mut blob: Vec = vec![0; cert_blob_info.blob_header.data_length as usize]; + // data_length measurs in field element, multiply to get num bytes + let mut blob: Vec = + vec![0; cert_blob_info.blob_header.data_length as usize * BYTES_PER_FIELD_ELEMENT]; // 96 because our g1 commitment has 64 bytes in v1 // why 96, the original 4844 has bytes length of 80 (it has 48 bytes for commitment) @@ -62,10 +64,8 @@ impl EigenDABlobProvider for OracleEigenDAProvider let mut blob_key = [0u8; 96]; // In eigenDA terminology, length describes the number of field element, size describes - // number of bytes. In eigenda proxy memstore mode, the datalength is wronly assigned to - // be the bytes lenght. We need to resolve it later. - // For now, we internally divides 32. ToDo - let data_length = cert_blob_info.blob_header.data_length as u64 / 32; + // number of bytes. + let data_length = cert_blob_info.blob_header.data_length as u64; info!("cert_blob_info.blob_header.data_length {:?}", data_length); diff --git a/op-devnet.docker-compose.yml.patch b/op-devnet.docker-compose.yml.patch index 83bc7cb..cda670d 100644 --- a/op-devnet.docker-compose.yml.patch +++ b/op-devnet.docker-compose.yml.patch @@ -18,7 +18,7 @@ index adcaea8f4..5c5e2e8ee 100644 - --port=3100 - --log.level=debug - --generic-commitment="${ALTDA_GENERIC_DA}" -+ image: ghcr.io/layr-labs/eigenda-proxy:v1.6.1 ++ image: ghcr.io/layr-labs/eigenda-proxy:main + environment: + EIGENDA_PROXY_ADDR: 0.0.0.0 + EIGENDA_PROXY_PORT: 3100 From d0b988d93953f2fd75f89632dcff611b3effba35 Mon Sep 17 00:00:00 2001 From: Bowen Date: Fri, 3 Jan 2025 13:55:30 -0800 Subject: [PATCH 07/11] rm unneeded deps --- Cargo.lock | 1 - bin/host/Cargo.toml | 3 --- 2 files changed, 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ff6e14d..99d3006 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1978,7 +1978,6 @@ dependencies = [ "kona-preimage", "proptest", "reqwest", - "rust-kzg-bn254", "tokio", "tracing", ] diff --git a/bin/host/Cargo.toml b/bin/host/Cargo.toml index 73cedb7..69820d0 100644 --- a/bin/host/Cargo.toml +++ b/bin/host/Cargo.toml @@ -26,8 +26,5 @@ async-trait.workspace = true tokio = { workspace = true, features = ["full"] } clap = { workspace = true, features = ["derive", "env"] } -# Cryptography -rust-kzg-bn254.workspace = true - [dev-dependencies] proptest.workspace = true From 59b0c34a5a4d9b58598cc5ebd3773976382b692c Mon Sep 17 00:00:00 2001 From: Bowen Date: Fri, 3 Jan 2025 13:57:05 -0800 Subject: [PATCH 08/11] rm unneeded test --- crates/eigenda/src/eigenda_data.rs | 38 ------------------------------ 1 file changed, 38 deletions(-) diff --git a/crates/eigenda/src/eigenda_data.rs b/crates/eigenda/src/eigenda_data.rs index a7cdffe..67f256b 100644 --- a/crates/eigenda/src/eigenda_data.rs +++ b/crates/eigenda/src/eigenda_data.rs @@ -89,44 +89,6 @@ mod tests { use alloy_primitives::Bytes; use kona_derive::errors::BlobDecodingError; - fn generate_blob_data(content: &[u8]) -> EigenDABlobData { - let mut blob = vec![0; 32]; - blob[1] = BLOB_ENCODING_VERSION_0; - blob[2..6].copy_from_slice(&(content.len() as u32).to_be_bytes()); - blob.extend_from_slice(&helpers::convert_by_padding_empty_byte(content)); - EigenDABlobData { - blob: Bytes::from(blob), - } - } - - #[test] - fn test_decode_success() { - let content = vec![1, 2, 3, 4]; - let data = generate_blob_data(&content); - let result = data.decode(); - assert!(result.is_ok()); - assert_eq!(result.unwrap(), Bytes::from(content)); - } - - #[test] - fn test_decode_success_empty() { - let content = vec![]; - let data = generate_blob_data(&content); - let result = data.decode(); - assert!(result.is_ok()); - assert_eq!(result.unwrap(), Bytes::from(content)); - } - - #[test] - fn test_decode_error_invalid_length() { - let data = EigenDABlobData { - blob: Bytes::from(vec![0; 31]), // one byte short of having a full header - }; - let result = data.decode(); - assert!(result.is_err()); - assert_eq!(result.unwrap_err(), BlobDecodingError::InvalidLength); - } - #[test] fn test_encode_and_decode_success() { let rollup_data = vec![1, 2, 3, 4]; From 0adca56a61d8b7c6da899968457c330f7443bc29 Mon Sep 17 00:00:00 2001 From: Bowen Date: Fri, 3 Jan 2025 14:03:26 -0800 Subject: [PATCH 09/11] fix power of 2 bug --- bin/host/src/eigenda_fetcher/mod.rs | 5 ----- crates/eigenda/src/eigenda_data.rs | 16 +++++++--------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 14bbfe2..a0a1dc6 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -155,12 +155,7 @@ where let item_slice = cert.as_ref(); let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..]).unwrap(); - // TODO ensure data_length is always power of 2. Proxy made mistake // Proxy should return a cert whose data_length measured in symbol (i.e. 32 Bytes) - // Currently, it returns number of bytes. We need to fix proxy and here later. - //let data_size = cert_blob_info.blob_header.data_length as u64; - //let blob_length: u64 = data_size / 32; - let blob_length = cert_blob_info.blob_header.data_length as u64; warn!("blob length: {:?}", blob_length); diff --git a/crates/eigenda/src/eigenda_data.rs b/crates/eigenda/src/eigenda_data.rs index 67f256b..170ba73 100644 --- a/crates/eigenda/src/eigenda_data.rs +++ b/crates/eigenda/src/eigenda_data.rs @@ -1,4 +1,4 @@ -use crate::BLOB_ENCODING_VERSION_0; +use crate::{BLOB_ENCODING_VERSION_0, BYTES_PER_FIELD_ELEMENT}; use alloc::vec; use alloy_primitives::Bytes; use bytes::buf::Buf; @@ -50,11 +50,8 @@ impl EigenDABlobData { /// This matches exactly the eigenda proxy implementation, whose logic is in /// /// - /// The length of (header + payload) by the encode function is always power of 2 + /// The length of (header + payload) by the encode function is always multiple of 32 /// The eigenda proxy does not take such constraint. - /// TODO it is possible to remove such power of 2 constraint, such that the client is not - /// relying on the data_length from eigenda cert. It might save some comm rounds between - /// host and client. pub fn encode(rollup_data: &[u8]) -> Self { let rollup_data_size = rollup_data.len() as u32; @@ -64,7 +61,10 @@ impl EigenDABlobData { let blob_payload_size = codec_rollup_data.len(); let blob_size = blob_payload_size + 32; - let blob_size = blob_size.next_power_of_two(); + + // round up to the closest multiple of 32 + let blob_size = (blob_size + BYTES_PER_FIELD_ELEMENT - 1) / BYTES_PER_FIELD_ELEMENT + * BYTES_PER_FIELD_ELEMENT; let mut raw_blob = vec![0u8; blob_size as usize]; @@ -82,8 +82,6 @@ impl EigenDABlobData { #[cfg(test)] mod tests { - use crate::BLOB_ENCODING_VERSION_0; - use super::*; use alloc::vec; use alloy_primitives::Bytes; @@ -94,7 +92,7 @@ mod tests { let rollup_data = vec![1, 2, 3, 4]; let eigenda_blob = EigenDABlobData::encode(&rollup_data); let data_len = eigenda_blob.blob.len(); - assert!(data_len.is_power_of_two()); + assert!(data_len % BYTES_PER_FIELD_ELEMENT == 0); let result = eigenda_blob.decode(); assert!(result.is_ok()); From afe46e5e37b2b804043624e002cf4a147a74b07d Mon Sep 17 00:00:00 2001 From: Bowen Date: Fri, 3 Jan 2025 14:18:02 -0800 Subject: [PATCH 10/11] fix lint --- crates/eigenda/src/eigenda_data.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/eigenda/src/eigenda_data.rs b/crates/eigenda/src/eigenda_data.rs index 170ba73..d77a88b 100644 --- a/crates/eigenda/src/eigenda_data.rs +++ b/crates/eigenda/src/eigenda_data.rs @@ -60,11 +60,11 @@ impl EigenDABlobData { let blob_payload_size = codec_rollup_data.len(); - let blob_size = blob_payload_size + 32; + // the first field element contains the header + let blob_size = blob_payload_size + BYTES_PER_FIELD_ELEMENT; // round up to the closest multiple of 32 - let blob_size = (blob_size + BYTES_PER_FIELD_ELEMENT - 1) / BYTES_PER_FIELD_ELEMENT - * BYTES_PER_FIELD_ELEMENT; + let blob_size = blob_size.div_ceil(BYTES_PER_FIELD_ELEMENT) * BYTES_PER_FIELD_ELEMENT; let mut raw_blob = vec![0u8; blob_size as usize]; @@ -72,7 +72,8 @@ impl EigenDABlobData { raw_blob[2..6].copy_from_slice(&rollup_data_size.to_be_bytes()); // encode length as uint32 - raw_blob[32..(32 + blob_payload_size as usize)].copy_from_slice(&codec_rollup_data); + raw_blob[BYTES_PER_FIELD_ELEMENT..(BYTES_PER_FIELD_ELEMENT + blob_payload_size as usize)] + .copy_from_slice(&codec_rollup_data); Self { blob: Bytes::from(raw_blob), From 4460e72903d4b83720bc8d33beb69b5fd5698736 Mon Sep 17 00:00:00 2001 From: Bowen Date: Fri, 3 Jan 2025 15:02:02 -0800 Subject: [PATCH 11/11] add todo --- op-devnet.docker-compose.yml.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/op-devnet.docker-compose.yml.patch b/op-devnet.docker-compose.yml.patch index cda670d..b5411a8 100644 --- a/op-devnet.docker-compose.yml.patch +++ b/op-devnet.docker-compose.yml.patch @@ -18,7 +18,7 @@ index adcaea8f4..5c5e2e8ee 100644 - --port=3100 - --log.level=debug - --generic-commitment="${ALTDA_GENERIC_DA}" -+ image: ghcr.io/layr-labs/eigenda-proxy:main ++ image: ghcr.io/layr-labs/eigenda-proxy:main # TODO update image to v1.6.2 once this PR is released https://github.com/Layr-Labs/hokulea/pull/22 + environment: + EIGENDA_PROXY_ADDR: 0.0.0.0 + EIGENDA_PROXY_PORT: 3100