diff --git a/api/proto/common/common.proto b/api/proto/common/common.proto index a75327ed7..07390c692 100644 --- a/api/proto/common/common.proto +++ b/api/proto/common/common.proto @@ -2,6 +2,9 @@ syntax = "proto3"; package common; option go_package = "github.com/Layr-Labs/eigenda/api/grpc/common"; +// G1Commitment represents the serialized coordinates of a G1 KZG commitment. +// We use gnark-crypto so adopt its serialization, which is big-endian. See: +// https://github.com/Consensys/gnark-crypto/blob/779e884dabb38b92e677f4891286637a3d2e5734/ecc/bn254/fp/element.go#L862 message G1Commitment { // The X coordinate of the KZG commitment. This is the raw byte representation of the field element. bytes x = 1; @@ -10,11 +13,20 @@ message G1Commitment { } // BlobCommitment represents commitment of a specific blob, containing its -// KZG commitment, degree proof, the actual degree, and data length in number of symbols. +// KZG commitment, degree proof, the actual degree, and data length in number of symbols (field elements). +// It deserializes into https://github.com/Layr-Labs/eigenda/blob/ce89dab18d2f8f55004002e17dd3a18529277845/encoding/data.go#L27 +// +// See https://github.com/Layr-Labs/eigenda/blob/master/docs/spec/attestation/encoding.md#validation-via-kzg +// to understand how this commitment is used to validate the blob. message BlobCommitment { + // Concatenation of the x and y coordinates of `common.G1Commitment`. bytes commitment = 1; + // Serialization of the G2Commitment to the blob length. bytes length_commitment = 2; + // Serialization of the G2Affine element representing the proof of the blob length. bytes length_proof = 3; + // The length of the blob in symbols (field elements). + // TODO: is this length always a power of 2? Are there any other characteristics that we should list? etc. uint32 length = 4; } diff --git a/api/proto/disperser/v2/disperser_v2.proto b/api/proto/disperser/v2/disperser_v2.proto index 151c7523f..e037274f9 100644 --- a/api/proto/disperser/v2/disperser_v2.proto +++ b/api/proto/disperser/v2/disperser_v2.proto @@ -18,21 +18,40 @@ service Disperser { rpc GetBlobStatus(BlobStatusRequest) returns (BlobStatusReply) {} // GetBlobCommitment is a utility method that calculates commitment for a blob payload. + // It is provided to help clients who are trying to construct a DisperseBlobRequest.blob_header + // and don't have the ability to calculate the commitment themselves (expensive operation which requires SRS points). + // + // For an example usage, see how our disperser_client makes a call to this endpoint when it doesn't have a local prover: + // https://github.com/Layr-Labs/eigenda/blob/6059c6a068298d11c41e50f5bcd208d0da44906a/api/clients/v2/disperser_client.go#L166 rpc GetBlobCommitment(BlobCommitmentRequest) returns (BlobCommitmentReply) {} - // GetPaymentState is a utility method to get the payment state of a given account. + // GetPaymentState is a utility method to get the payment state of a given account, at a given disperser. + // EigenDA's payment system for v2 is currently centralized, meaning that each disperser does its own accounting. + // A client wanting to disperse a blob would thus need to synchronize its local accounting state with that of the disperser. + // That typically only needs to be done once, and the state can be updated locally as the client disperses blobs. + // The accounting rules are simple and can be updated locally, but periodic checks with the disperser can't hurt. + // + // For an example usage, see how our disperser_client makes a call to this endpoint to populate its local accountant struct: + // https://github.com/Layr-Labs/eigenda/blob/6059c6a068298d11c41e50f5bcd208d0da44906a/api/clients/v2/disperser_client.go#L298 rpc GetPaymentState(GetPaymentStateRequest) returns (GetPaymentStateReply) {} } // Requests and Replys message DisperseBlobRequest { - // The data to be dispersed. - // The size of data must be <= 16MiB. Every 32 bytes of data is interpreted as an integer in big endian format - // where the lower address has more significant bits. The integer must stay in the valid range to be interpreted - // as a field element on the bn254 curve. The valid range is - // 0 <= x < 21888242871839275222246405745257275088548364400416034343698204186575808495617 - // If any one of the 32 bytes elements is outside the range, the whole request is deemed as invalid, and rejected. + // The encoded data to be dispersed to the EigenDA network. + // + // Validation rules: + // 1. The size of data must be <= 16MiB. + // 2. The data is allowed to not be a multiple of 32 bytes: the last chunk will be padded with zeros to make it so. + // 3. Every 32 bytes chunk (including the last after rule 2) must be a valid bid-endian serialized field element on the bn254 curve. + // The valid range for each 32 byte chunk is: 0 <= x < 21888242871839275222246405745257275088548364400416034343698204186575808495617 + // If rule 1 or 3 is violated, the whole request is deemed as invalid, and rejected. + // + // To encode your payload data into the correct blob format, you can make use of our codec: + // https://github.com/Layr-Labs/eigenda/blob/82192985a2d15b88d85a6090404b2595f4922bef/api/clients/codecs/default_blob_codec.go#L21 + // Most users will not need to interact with this low level codec directly however, given that the high-level eigenda_client does the encoding for you: + // https://github.com/Layr-Labs/eigenda/blob/master/api/clients/eigenda_client.go bytes data = 1; common.v2.BlobHeader blob_header = 2; } diff --git a/encoding/encoding.go b/encoding/encoding.go index b4af4ac2d..ef29e6c6b 100644 --- a/encoding/encoding.go +++ b/encoding/encoding.go @@ -12,6 +12,9 @@ type Prover interface { // reconstruct the blob. EncodeAndProve(data []byte, params EncodingParams) (BlobCommitments, []*Frame, error) + // GetCommitmentsForPaddedLength takes in a byte slice representing a list of bn254 + // field elements (32 bytes each, except potentially the last element), + // pads the (potentially incomplete) last element with zeroes, and returns the commitments for the padded list. GetCommitmentsForPaddedLength(data []byte) (BlobCommitments, error) GetFrames(data []byte, params EncodingParams) ([]*Frame, error) diff --git a/encoding/kzg/prover/prover.go b/encoding/kzg/prover/prover.go index bdcce2ab9..041323ed5 100644 --- a/encoding/kzg/prover/prover.go +++ b/encoding/kzg/prover/prover.go @@ -212,6 +212,9 @@ func (e *Prover) GetFrames(data []byte, params encoding.EncodingParams) ([]*enco return chunks, nil } +// GetCommitmentsForPaddedLength takes in a byte slice representing a list of bn254 +// field elements (32 bytes each, except potentially the last element), +// pads the (potentially incomplete) last element with zeroes, and returns the commitments for the padded list. func (e *Prover) GetCommitmentsForPaddedLength(data []byte) (encoding.BlobCommitments, error) { symbols, err := rs.ToFrArray(data) if err != nil { diff --git a/encoding/rs/utils.go b/encoding/rs/utils.go index d1959380b..0895a533b 100644 --- a/encoding/rs/utils.go +++ b/encoding/rs/utils.go @@ -10,6 +10,15 @@ import ( "github.com/consensys/gnark-crypto/ecc/bn254/fr" ) +// ToFrArray deserializes a byte array into a list of bn254 field elements., +// where each 32-byte chunk needs to be a big-endian serialized bn254 field element. +// The last chunk is allowed to not have 32-bytes, and will be padded with zeroes +// on the right (so make sure that the last partial chunk represents a valid field element +// when padded with zeroes on the right and interpreted as big-endian). +// +// TODO: we should probably just force the data to be a multiple of 32 bytes. +// This would make the API and code simpler to read, and also allow the code +// to be auto-vectorized by the compiler (it probably isn't right now given the if inside the for loop). func ToFrArray(data []byte) ([]fr.Element, error) { numEle := GetNumElement(uint64(len(data)), encoding.BYTES_PER_SYMBOL) eles := make([]fr.Element, numEle) @@ -35,7 +44,9 @@ func ToFrArray(data []byte) ([]fr.Element, error) { return eles, nil } -// ToByteArray converts a list of Fr to a byte array +// ToByteArray serializes a slice of fields elements to a slice of bytes. +// The byte array is created by serializing each Fr element in big-endian format. +// It is the reverse operation of ToFrArray. func ToByteArray(dataFr []fr.Element, maxDataSize uint64) []byte { n := len(dataFr) dataSize := int(math.Min(