diff --git a/encoding.go b/encoding.go index d5e27d17..0f64e682 100644 --- a/encoding.go +++ b/encoding.go @@ -56,12 +56,11 @@ const ( leafC1CommitmentOffset = leafCommitmentOffset + banderwagon.UncompressedSize leafC2CommitmentOffset = leafC1CommitmentOffset + banderwagon.UncompressedSize leafChildrenOffset = leafC2CommitmentOffset + banderwagon.UncompressedSize - leafBalanceSize = 32 - leafNonceSize = 8 + leafBasicDataSize = 32 leafSlotSize = 32 leafValueIndexSize = 1 singleSlotLeafSize = nodeTypeSize + StemSize + 2*banderwagon.UncompressedSize + leafValueIndexSize + leafSlotSize - eoaLeafSize = nodeTypeSize + StemSize + 2*banderwagon.UncompressedSize + leafBalanceSize + leafNonceSize + eoaLeafSize = nodeTypeSize + StemSize + 2*banderwagon.UncompressedSize + leafBasicDataSize ) func bit(bitlist []byte, nr int) bool { @@ -94,6 +93,8 @@ func ParseNode(serializedNode []byte, depth byte) (VerkleNode, error) { return parseEoAccountNode(serializedNode, depth) case singleSlotType: return parseSingleSlotNode(serializedNode, depth) + case skipListType: + return parseSkipList(serializedNode, depth) default: return nil, ErrInvalidNodeEncoding } @@ -135,17 +136,49 @@ func parseLeafNode(serialized []byte, depth byte) (VerkleNode, error) { return ln, nil } +func parseSkipList(serialized []byte, depth byte) (VerkleNode, error) { + var values [NodeWidth][]byte + offset := leafStemOffset + StemSize + 3*banderwagon.UncompressedSize // offset in the serialized payload + valueIdx := 0 // Index of the value being deserialized + for valueIdx < NodeWidth { + rangecount := serialized[offset+1] + gapsize := serialized[offset] + valueIdx += int(gapsize) + offset += 2 + for i := 0; i < int(rangecount); i++ { + values[valueIdx] = serialized[offset : offset+leafSlotSize] + offset += leafSlotSize + valueIdx++ + } + } + ln := NewLeafNodeWithNoComms(serialized[leafStemOffset:leafStemOffset+StemSize], values[:]) + ln.setDepth(depth) + ln.c1 = new(Point) + + // Sanity check that we have at least 3*banderwagon.UncompressedSize bytes left in the serialized payload. + if len(serialized[leafCommitmentOffset:]) < 3*banderwagon.UncompressedSize { + return nil, fmt.Errorf("leaf node commitments are not the correct size, expected at least %d, got %d", 3*banderwagon.UncompressedSize, len(serialized[leafC1CommitmentOffset:])) + } + + if err := ln.c1.SetBytesUncompressed(serialized[leafC1CommitmentOffset:leafC1CommitmentOffset+banderwagon.UncompressedSize], true); err != nil { + return nil, fmt.Errorf("setting c1 commitment: %w", err) + } + ln.c2 = new(Point) + if err := ln.c2.SetBytesUncompressed(serialized[leafC2CommitmentOffset:leafC2CommitmentOffset+banderwagon.UncompressedSize], true); err != nil { + return nil, fmt.Errorf("setting c2 commitment: %w", err) + } + ln.commitment = new(Point) + if err := ln.commitment.SetBytesUncompressed(serialized[leafCommitmentOffset:leafC1CommitmentOffset], true); err != nil { + return nil, fmt.Errorf("setting commitment: %w", err) + } + return ln, nil +} + func parseEoAccountNode(serialized []byte, depth byte) (VerkleNode, error) { var values [NodeWidth][]byte offset := leafStemOffset + StemSize + 2*banderwagon.UncompressedSize - values[0] = zero32[:] // 0 version - values[1] = serialized[offset : offset+leafBalanceSize] // balance - var nonce [32]byte - offset += leafBalanceSize - copy(nonce[:leafNonceSize], serialized[offset:offset+leafNonceSize]) - values[2] = nonce[:] // nonce - values[3] = EmptyCodeHash[:] - values[4] = zero32[:] // 0 code size + values[0] = serialized[offset : offset+leafBasicDataSize] // basic data + values[1] = EmptyCodeHash[:] ln := NewLeafNodeWithNoComms(serialized[leafStemOffset:leafStemOffset+StemSize], values[:]) ln.setDepth(depth) ln.c1 = new(Point) diff --git a/encoding_test.go b/encoding_test.go index c7eb53d6..3f6ed4bd 100644 --- a/encoding_test.go +++ b/encoding_test.go @@ -2,6 +2,7 @@ package verkle import ( "bytes" + "encoding/binary" "testing" "github.com/crate-crypto/go-ipa/banderwagon" @@ -22,7 +23,9 @@ func TestLeafStemLength(t *testing.T) { // Serialize a leaf with no values, but whose stem is 32 bytes. The // serialization should trim the extra byte. toolong := make([]byte, 32) - leaf, err := NewLeafNode(toolong, make([][]byte, NodeWidth)) + values := make([][]byte, NodeWidth) + values[42] = zero32[:] + leaf, err := NewLeafNode(toolong, values) if err != nil { t.Fatal(err) } @@ -30,8 +33,8 @@ func TestLeafStemLength(t *testing.T) { if err != nil { t.Fatal(err) } - if len(ser) != nodeTypeSize+StemSize+bitlistSize+3*banderwagon.UncompressedSize { - t.Fatalf("invalid serialization when the stem is longer than 31 bytes: %x (%d bytes != %d)", ser, len(ser), nodeTypeSize+StemSize+bitlistSize+2*banderwagon.UncompressedSize) + if len(ser) != singleSlotLeafSize { + t.Fatalf("invalid serialization when the stem is longer than 31 bytes: %x (%d bytes != %d)", ser, len(ser), singleSlotLeafSize) } } @@ -61,12 +64,11 @@ func TestInvalidNodeEncoding(t *testing.T) { } func TestParseNodeEoA(t *testing.T) { + var basicdata [32]byte values := make([][]byte, 256) - values[0] = zero32[:] + values[0] = basicdata[:] + binary.BigEndian.PutUint64(values[0][8:], 0xde) values[1] = EmptyCodeHash[:] // set empty code hash as balance, because why not - values[2] = fourtyKeyTest[:] // set nonce to 64 - values[3] = EmptyCodeHash[:] // set empty code hash - values[4] = zero32[:] // zero-size ln, err := NewLeafNode(ffx32KeyTest[:31], values) if err != nil { t.Fatalf("error creating leaf node: %v", err) @@ -99,26 +101,15 @@ func TestParseNodeEoA(t *testing.T) { t.Fatalf("invalid stem, got %x, expected %x", lnd.stem, ffx32KeyTest[:31]) } - if !bytes.Equal(lnd.values[0], zero32[:]) { - t.Fatalf("invalid version, got %x, expected %x", lnd.values[0], zero32[:]) + nonce := binary.BigEndian.Uint64(lnd.values[0][8:]) + if nonce != 0xde { + t.Fatalf("invalid version, got %x, expected %x", nonce, 0xde) } if !bytes.Equal(lnd.values[1], EmptyCodeHash[:]) { t.Fatalf("invalid balance, got %x, expected %x", lnd.values[1], EmptyCodeHash[:]) } - if !bytes.Equal(lnd.values[2], fourtyKeyTest[:]) { - t.Fatalf("invalid nonce, got %x, expected %x", lnd.values[2], fourtyKeyTest[:]) - } - - if !bytes.Equal(lnd.values[3], EmptyCodeHash[:]) { - t.Fatalf("invalid code hash, got %x, expected %x", lnd.values[3], EmptyCodeHash[:]) - } - - if !bytes.Equal(lnd.values[4], zero32[:]) { - t.Fatalf("invalid code size, got %x, expected %x", lnd.values[4], zero32[:]) - } - if !lnd.c2.Equal(&banderwagon.Identity) { t.Fatalf("invalid c2, got %x, expected %x", lnd.c2, banderwagon.Identity) } @@ -190,3 +181,79 @@ func TestParseNodeSingleSlot(t *testing.T) { t.Fatalf("invalid commitment, got %x, expected %x", lnd.commitment, ln.commitment) } } + +func TestSerializeWithSkipLists(t *testing.T) { + t.Parallel() + + values := make([][]byte, NodeWidth) + values[42] = zero32[:] + values[57] = fourtyKeyTest[:] + leaf, err := NewLeafNode(ffx32KeyTest, values) + if err != nil { + t.Fatal(err) + } + ser, err := leaf.Serialize() + if err != nil { + t.Fatal(err) + } + if len(ser) == 0 { + t.Fatal("empty serialization buffer") + } + if ser[0] != skipListType { + t.Fatalf("invalid serialization type, got %d, expected %d", ser[0], skipListType) + } + if !bytes.Equal(ser[1:32], ffx32KeyTest[:31]) { + t.Fatalf("stem didn't serialize properly, got %x, want %x", ser[1:32], ffx32KeyTest[:31]) + } + expectedSize := nodeTypeSize + StemSize + 3*banderwagon.UncompressedSize + 4 + 2*leafSlotSize + if len(ser) != expectedSize { + t.Fatalf("invalid skiplist serialization: %x (%d bytes != %d)", ser, len(ser), expectedSize) + } + if ser[nodeTypeSize+StemSize+3*banderwagon.UncompressedSize] != 42 { + t.Fatalf("invalid amount of leaves skipped, got %d, want %d", ser[nodeTypeSize+StemSize+3*banderwagon.UncompressedSize], 42) + } + if ser[nodeTypeSize+StemSize+3*banderwagon.UncompressedSize+1] != 1 { + t.Fatalf("invalid amount of leaves skipped, got %d, want %d", ser[nodeTypeSize+StemSize+3*banderwagon.UncompressedSize+1], 42) + } + if ser[nodeTypeSize+StemSize+3*banderwagon.UncompressedSize+2+leafSlotSize] != 14 { + t.Fatalf("invalid amount of leaves skipped, got %d, want %d", ser[nodeTypeSize+StemSize+3*banderwagon.UncompressedSize+2+leafSlotSize], 14) + } + + // add a last value to check that the final gap is properly handled + values[255] = ffx32KeyTest + ser, err = leaf.Serialize() + if err != nil { + t.Fatal(err) + } + expectedSize = nodeTypeSize + StemSize + 3*banderwagon.UncompressedSize + 6 + 3*leafSlotSize + if len(ser) != expectedSize { + t.Fatalf("invalid skiplist serialization: %x (%d bytes != %d)", ser, len(ser), expectedSize) + } + + deser, err := ParseNode(ser, 5) + if err != nil { + t.Fatal(err) + } + vals := deser.(*LeafNode).values + for i, val := range vals { + + switch i { + case 42: + if !bytes.Equal(val, zero32[:]) { + t.Fatalf("invalid deserialized skiplist value at %d: got %x, want %x", i, val, zero32) + } + case 57: + if !bytes.Equal(val, fourtyKeyTest[:]) { + t.Fatalf("invalid deserialized skiplist value at %d: got %x, want %x", i, val, fourtyKeyTest) + } + case 255: + if !bytes.Equal(val, ffx32KeyTest[:]) { + t.Fatalf("invalid deserialized skiplist value at %d: got %x, want %x", i, val, ffx32KeyTest) + } + default: + if val != nil { + t.Fatalf("invalid deserialized skiplist value at %d: got %x, want nil", i, val) + } + } + } +} diff --git a/tree.go b/tree.go index f0788164..b35305ec 100644 --- a/tree.go +++ b/tree.go @@ -170,6 +170,7 @@ const ( leafType byte = 2 eoAccountType byte = 3 singleSlotType byte = 4 + skipListType byte = 8 ) type ( @@ -1775,35 +1776,42 @@ func (n *LeafNode) serializeLeafWithUncompressedCommitments(cBytes, c1Bytes, c2B bitlist [bitlistSize]byte isEoA = true count, lastIdx int + gapcount int + gaps [32]struct { + Skip byte // How many slots to skip before the next range + Count byte // Size of the next range + } ) for i, v := range n.values { if v != nil { count++ lastIdx = i + gaps[gapcount].Count++ + setBit(bitlist[:], i) children = append(children, v...) if padding := emptyValue[:LeafValueSize-len(v)]; len(padding) != 0 { children = append(children, padding...) } + } else { + if gaps[gapcount].Skip == 255 { + panic("empty leaf node") + } + if i > 0 && n.values[i-1] != nil { + gapcount++ + } + gaps[gapcount].Skip++ } + // Check for an EOA if isEoA { switch i { case 0: - // Version should be 0 - isEoA = v != nil && bytes.Equal(v, zero32[:]) - case 1: - // Balance should not be nil + // Basic data should not be nil isEoA = v != nil - case 2: - // Nonce should have its last 24 bytes set to 0 - isEoA = v != nil && bytes.Equal(v[leafNonceSize:], zero24[:]) - case 3: + case 1: // Code hash should be the empty code hash isEoA = v != nil && bytes.Equal(v, EmptyCodeHash[:]) - case 4: - // Code size must be 0 - isEoA = v != nil && bytes.Equal(v, zero32[:]) default: // All other values must be nil isEoA = v == nil @@ -1830,8 +1838,28 @@ func (n *LeafNode) serializeLeafWithUncompressedCommitments(cBytes, c1Bytes, c2B copy(result[leafStemOffset:], n.stem[:StemSize]) copy(result[leafStemOffset+StemSize:], c1Bytes[:]) copy(result[leafStemOffset+StemSize+banderwagon.UncompressedSize:], cBytes[:]) - copy(result[leafStemOffset+StemSize+2*banderwagon.UncompressedSize:], n.values[1]) // copy balance - copy(result[leafStemOffset+StemSize+2*banderwagon.UncompressedSize+leafBalanceSize:], n.values[2][:leafNonceSize]) // copy nonce + copy(result[leafStemOffset+StemSize+2*banderwagon.UncompressedSize:], n.values[0]) // copy basic data + case gapcount < 16: + // If there are less than 16 gaps, it's worth using skiplists + result = make([]byte, 1, nodeTypeSize+StemSize+bitlistSize+3*banderwagon.UncompressedSize+len(children)) + result[0] = skipListType + result = append(result, n.stem[:StemSize]...) + result = append(result, cBytes[:]...) + result = append(result, c1Bytes[:]...) + result = append(result, c2Bytes[:]...) + var leafIdx int + for _, gap := range gaps { + if gap.Count == 0 { + break // skip the last gap as nothing follows + } + result = append(result, gap.Skip) + leafIdx += int(gap.Skip) + result = append(result, gap.Count) + for i := 0; i < int(gap.Count); i++ { + result = append(result, n.values[leafIdx]...) + leafIdx++ + } + } default: result = make([]byte, nodeTypeSize+StemSize+bitlistSize+3*banderwagon.UncompressedSize+len(children)) result[0] = leafType