Skip to content

Commit

Permalink
use skiplists to compress leave storage
Browse files Browse the repository at this point in the history
  • Loading branch information
@gballet
gballet committed Sep 11, 2024
1 parent 722a15c commit 5444cd1
Show file tree
Hide file tree
Showing 3 changed files with 173 additions and 45 deletions.
55 changes: 44 additions & 11 deletions encoding.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,11 @@ const (
leafC1CommitmentOffset = leafCommitmentOffset + banderwagon.UncompressedSize
leafC2CommitmentOffset = leafC1CommitmentOffset + banderwagon.UncompressedSize
leafChildrenOffset = leafC2CommitmentOffset + banderwagon.UncompressedSize
leafBalanceSize = 32
leafNonceSize = 8
leafBasicDataSize = 32
leafSlotSize = 32
leafValueIndexSize = 1
singleSlotLeafSize = nodeTypeSize + StemSize + 2*banderwagon.UncompressedSize + leafValueIndexSize + leafSlotSize
eoaLeafSize = nodeTypeSize + StemSize + 2*banderwagon.UncompressedSize + leafBalanceSize + leafNonceSize
eoaLeafSize = nodeTypeSize + StemSize + 2*banderwagon.UncompressedSize + leafBasicDataSize
)

func bit(bitlist []byte, nr int) bool {
Expand Down Expand Up @@ -94,6 +93,8 @@ func ParseNode(serializedNode []byte, depth byte) (VerkleNode, error) {
return parseEoAccountNode(serializedNode, depth)
case singleSlotType:
return parseSingleSlotNode(serializedNode, depth)
case skipListType:
return parseSkipList(serializedNode, depth)
default:
return nil, ErrInvalidNodeEncoding
}
Expand Down Expand Up @@ -135,17 +136,49 @@ func parseLeafNode(serialized []byte, depth byte) (VerkleNode, error) {
return ln, nil
}

func parseSkipList(serialized []byte, depth byte) (VerkleNode, error) {
var values [NodeWidth][]byte
offset := leafStemOffset + StemSize + 3*banderwagon.UncompressedSize // offset in the serialized payload
valueIdx := 0 // Index of the value being deserialized
for valueIdx < NodeWidth {
rangecount := serialized[offset+1]
gapsize := serialized[offset]
valueIdx += int(gapsize)
offset += 2
for i := 0; i < int(rangecount); i++ {
values[valueIdx] = serialized[offset : offset+leafSlotSize]
offset += leafSlotSize
valueIdx++
}
}
ln := NewLeafNodeWithNoComms(serialized[leafStemOffset:leafStemOffset+StemSize], values[:])
ln.setDepth(depth)
ln.c1 = new(Point)

// Sanity check that we have at least 3*banderwagon.UncompressedSize bytes left in the serialized payload.
if len(serialized[leafCommitmentOffset:]) < 3*banderwagon.UncompressedSize {
return nil, fmt.Errorf("leaf node commitments are not the correct size, expected at least %d, got %d", 3*banderwagon.UncompressedSize, len(serialized[leafC1CommitmentOffset:]))
}

if err := ln.c1.SetBytesUncompressed(serialized[leafC1CommitmentOffset:leafC1CommitmentOffset+banderwagon.UncompressedSize], true); err != nil {
return nil, fmt.Errorf("setting c1 commitment: %w", err)
}
ln.c2 = new(Point)
if err := ln.c2.SetBytesUncompressed(serialized[leafC2CommitmentOffset:leafC2CommitmentOffset+banderwagon.UncompressedSize], true); err != nil {
return nil, fmt.Errorf("setting c2 commitment: %w", err)
}
ln.commitment = new(Point)
if err := ln.commitment.SetBytesUncompressed(serialized[leafCommitmentOffset:leafC1CommitmentOffset], true); err != nil {
return nil, fmt.Errorf("setting commitment: %w", err)
}
return ln, nil
}

func parseEoAccountNode(serialized []byte, depth byte) (VerkleNode, error) {
var values [NodeWidth][]byte
offset := leafStemOffset + StemSize + 2*banderwagon.UncompressedSize
values[0] = zero32[:] // 0 version
values[1] = serialized[offset : offset+leafBalanceSize] // balance
var nonce [32]byte
offset += leafBalanceSize
copy(nonce[:leafNonceSize], serialized[offset:offset+leafNonceSize])
values[2] = nonce[:] // nonce
values[3] = EmptyCodeHash[:]
values[4] = zero32[:] // 0 code size
values[0] = serialized[offset : offset+leafBasicDataSize] // basic data
values[1] = EmptyCodeHash[:]
ln := NewLeafNodeWithNoComms(serialized[leafStemOffset:leafStemOffset+StemSize], values[:])
ln.setDepth(depth)
ln.c1 = new(Point)
Expand Down
109 changes: 88 additions & 21 deletions encoding_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package verkle

import (
"bytes"
"encoding/binary"
"testing"

"github.com/crate-crypto/go-ipa/banderwagon"
Expand All @@ -22,16 +23,18 @@ func TestLeafStemLength(t *testing.T) {
// Serialize a leaf with no values, but whose stem is 32 bytes. The
// serialization should trim the extra byte.
toolong := make([]byte, 32)
leaf, err := NewLeafNode(toolong, make([][]byte, NodeWidth))
values := make([][]byte, NodeWidth)
values[42] = zero32[:]
leaf, err := NewLeafNode(toolong, values)
if err != nil {
t.Fatal(err)
}
ser, err := leaf.Serialize()
if err != nil {
t.Fatal(err)
}
if len(ser) != nodeTypeSize+StemSize+bitlistSize+3*banderwagon.UncompressedSize {
t.Fatalf("invalid serialization when the stem is longer than 31 bytes: %x (%d bytes != %d)", ser, len(ser), nodeTypeSize+StemSize+bitlistSize+2*banderwagon.UncompressedSize)
if len(ser) != singleSlotLeafSize {
t.Fatalf("invalid serialization when the stem is longer than 31 bytes: %x (%d bytes != %d)", ser, len(ser), singleSlotLeafSize)
}
}

Expand Down Expand Up @@ -61,12 +64,11 @@ func TestInvalidNodeEncoding(t *testing.T) {
}

func TestParseNodeEoA(t *testing.T) {
var basicdata [32]byte
values := make([][]byte, 256)
values[0] = zero32[:]
values[0] = basicdata[:]
binary.BigEndian.PutUint64(values[0][8:], 0xde)
values[1] = EmptyCodeHash[:] // set empty code hash as balance, because why not
values[2] = fourtyKeyTest[:] // set nonce to 64
values[3] = EmptyCodeHash[:] // set empty code hash
values[4] = zero32[:] // zero-size
ln, err := NewLeafNode(ffx32KeyTest[:31], values)
if err != nil {
t.Fatalf("error creating leaf node: %v", err)
Expand Down Expand Up @@ -99,26 +101,15 @@ func TestParseNodeEoA(t *testing.T) {
t.Fatalf("invalid stem, got %x, expected %x", lnd.stem, ffx32KeyTest[:31])
}

if !bytes.Equal(lnd.values[0], zero32[:]) {
t.Fatalf("invalid version, got %x, expected %x", lnd.values[0], zero32[:])
nonce := binary.BigEndian.Uint64(lnd.values[0][8:])
if nonce != 0xde {
t.Fatalf("invalid version, got %x, expected %x", nonce, 0xde)
}

if !bytes.Equal(lnd.values[1], EmptyCodeHash[:]) {
t.Fatalf("invalid balance, got %x, expected %x", lnd.values[1], EmptyCodeHash[:])
}

if !bytes.Equal(lnd.values[2], fourtyKeyTest[:]) {
t.Fatalf("invalid nonce, got %x, expected %x", lnd.values[2], fourtyKeyTest[:])
}

if !bytes.Equal(lnd.values[3], EmptyCodeHash[:]) {
t.Fatalf("invalid code hash, got %x, expected %x", lnd.values[3], EmptyCodeHash[:])
}

if !bytes.Equal(lnd.values[4], zero32[:]) {
t.Fatalf("invalid code size, got %x, expected %x", lnd.values[4], zero32[:])
}

if !lnd.c2.Equal(&banderwagon.Identity) {
t.Fatalf("invalid c2, got %x, expected %x", lnd.c2, banderwagon.Identity)
}
Expand Down Expand Up @@ -190,3 +181,79 @@ func TestParseNodeSingleSlot(t *testing.T) {
t.Fatalf("invalid commitment, got %x, expected %x", lnd.commitment, ln.commitment)
}
}

func TestSerializeWithSkipLists(t *testing.T) {
t.Parallel()

values := make([][]byte, NodeWidth)
values[42] = zero32[:]
values[57] = fourtyKeyTest[:]
leaf, err := NewLeafNode(ffx32KeyTest, values)
if err != nil {
t.Fatal(err)
}
ser, err := leaf.Serialize()
if err != nil {
t.Fatal(err)
}
if len(ser) == 0 {
t.Fatal("empty serialization buffer")
}
if ser[0] != skipListType {
t.Fatalf("invalid serialization type, got %d, expected %d", ser[0], skipListType)
}
if !bytes.Equal(ser[1:32], ffx32KeyTest[:31]) {
t.Fatalf("stem didn't serialize properly, got %x, want %x", ser[1:32], ffx32KeyTest[:31])
}
expectedSize := nodeTypeSize + StemSize + 3*banderwagon.UncompressedSize + 4 + 2*leafSlotSize
if len(ser) != expectedSize {
t.Fatalf("invalid skiplist serialization: %x (%d bytes != %d)", ser, len(ser), expectedSize)
}
if ser[nodeTypeSize+StemSize+3*banderwagon.UncompressedSize] != 42 {
t.Fatalf("invalid amount of leaves skipped, got %d, want %d", ser[nodeTypeSize+StemSize+3*banderwagon.UncompressedSize], 42)
}
if ser[nodeTypeSize+StemSize+3*banderwagon.UncompressedSize+1] != 1 {
t.Fatalf("invalid amount of leaves skipped, got %d, want %d", ser[nodeTypeSize+StemSize+3*banderwagon.UncompressedSize+1], 42)
}
if ser[nodeTypeSize+StemSize+3*banderwagon.UncompressedSize+2+leafSlotSize] != 14 {
t.Fatalf("invalid amount of leaves skipped, got %d, want %d", ser[nodeTypeSize+StemSize+3*banderwagon.UncompressedSize+2+leafSlotSize], 14)
}

// add a last value to check that the final gap is properly handled
values[255] = ffx32KeyTest
ser, err = leaf.Serialize()
if err != nil {
t.Fatal(err)
}
expectedSize = nodeTypeSize + StemSize + 3*banderwagon.UncompressedSize + 6 + 3*leafSlotSize
if len(ser) != expectedSize {
t.Fatalf("invalid skiplist serialization: %x (%d bytes != %d)", ser, len(ser), expectedSize)
}

deser, err := ParseNode(ser, 5)
if err != nil {
t.Fatal(err)
}
vals := deser.(*LeafNode).values
for i, val := range vals {

switch i {
case 42:
if !bytes.Equal(val, zero32[:]) {
t.Fatalf("invalid deserialized skiplist value at %d: got %x, want %x", i, val, zero32)
}
case 57:
if !bytes.Equal(val, fourtyKeyTest[:]) {
t.Fatalf("invalid deserialized skiplist value at %d: got %x, want %x", i, val, fourtyKeyTest)
}
case 255:
if !bytes.Equal(val, ffx32KeyTest[:]) {
t.Fatalf("invalid deserialized skiplist value at %d: got %x, want %x", i, val, ffx32KeyTest)
}
default:
if val != nil {
t.Fatalf("invalid deserialized skiplist value at %d: got %x, want nil", i, val)
}
}
}
}
54 changes: 41 additions & 13 deletions tree.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ const (
leafType byte = 2
eoAccountType byte = 3
singleSlotType byte = 4
skipListType byte = 8
)

type (
Expand Down Expand Up @@ -1775,35 +1776,42 @@ func (n *LeafNode) serializeLeafWithUncompressedCommitments(cBytes, c1Bytes, c2B
bitlist [bitlistSize]byte
isEoA = true
count, lastIdx int
gapcount int
gaps [32]struct {
Skip byte // How many slots to skip before the next range
Count byte // Size of the next range
}
)
for i, v := range n.values {
if v != nil {
count++
lastIdx = i
gaps[gapcount].Count++

setBit(bitlist[:], i)
children = append(children, v...)
if padding := emptyValue[:LeafValueSize-len(v)]; len(padding) != 0 {
children = append(children, padding...)
}
} else {
if gaps[gapcount].Skip == 255 {
panic("empty leaf node")
}
if i > 0 && n.values[i-1] != nil {
gapcount++
}
gaps[gapcount].Skip++
}

// Check for an EOA
if isEoA {
switch i {
case 0:
// Version should be 0
isEoA = v != nil && bytes.Equal(v, zero32[:])
case 1:
// Balance should not be nil
// Basic data should not be nil
isEoA = v != nil
case 2:
// Nonce should have its last 24 bytes set to 0
isEoA = v != nil && bytes.Equal(v[leafNonceSize:], zero24[:])
case 3:
case 1:
// Code hash should be the empty code hash
isEoA = v != nil && bytes.Equal(v, EmptyCodeHash[:])
case 4:
// Code size must be 0
isEoA = v != nil && bytes.Equal(v, zero32[:])
default:
// All other values must be nil
isEoA = v == nil
Expand All @@ -1830,8 +1838,28 @@ func (n *LeafNode) serializeLeafWithUncompressedCommitments(cBytes, c1Bytes, c2B
copy(result[leafStemOffset:], n.stem[:StemSize])
copy(result[leafStemOffset+StemSize:], c1Bytes[:])
copy(result[leafStemOffset+StemSize+banderwagon.UncompressedSize:], cBytes[:])
copy(result[leafStemOffset+StemSize+2*banderwagon.UncompressedSize:], n.values[1]) // copy balance
copy(result[leafStemOffset+StemSize+2*banderwagon.UncompressedSize+leafBalanceSize:], n.values[2][:leafNonceSize]) // copy nonce
copy(result[leafStemOffset+StemSize+2*banderwagon.UncompressedSize:], n.values[0]) // copy basic data
case gapcount < 16:
// If there are less than 16 gaps, it's worth using skiplists
result = make([]byte, 1, nodeTypeSize+StemSize+bitlistSize+3*banderwagon.UncompressedSize+len(children))
result[0] = skipListType
result = append(result, n.stem[:StemSize]...)
result = append(result, cBytes[:]...)
result = append(result, c1Bytes[:]...)
result = append(result, c2Bytes[:]...)
var leafIdx int
for _, gap := range gaps {
if gap.Count == 0 {
break // skip the last gap as nothing follows
}
result = append(result, gap.Skip)
leafIdx += int(gap.Skip)
result = append(result, gap.Count)
for i := 0; i < int(gap.Count); i++ {
result = append(result, n.values[leafIdx]...)
leafIdx++
}
}
default:
result = make([]byte, nodeTypeSize+StemSize+bitlistSize+3*banderwagon.UncompressedSize+len(children))
result[0] = leafType
Expand Down

0 comments on commit 5444cd1

Please sign in to comment.