From d26ca1c2a2fc8ba1d68f4201fc33f85b5064d251 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Wed, 26 Apr 2023 12:51:21 -0300 Subject: [PATCH 1/8] tree: allow batching new leaf nodes Signed-off-by: Ignacio Hagopian tree: simplify batching of new leaves creation Signed-off-by: Ignacio Hagopian tree: fix insert new leaves test and benchmark Signed-off-by: Ignacio Hagopian remove comment Signed-off-by: Ignacio Hagopian remove conversion file Signed-off-by: Ignacio Hagopian remove unused method Signed-off-by: Ignacio Hagopian tree: add comment to explain different strategies on leaf node value updating Signed-off-by: Ignacio Hagopian --- conversion.go | 173 ------------------------------------------ encoding_test.go | 10 +-- tree.go | 193 +++++++++++++++++++++++++++++++++-------------- tree_test.go | 99 ++++-------------------- 4 files changed, 155 insertions(+), 320 deletions(-) delete mode 100644 conversion.go diff --git a/conversion.go b/conversion.go deleted file mode 100644 index 7e93f860..00000000 --- a/conversion.go +++ /dev/null @@ -1,173 +0,0 @@ -package verkle - -import ( - "bytes" - "context" - "fmt" - "runtime" - "sort" - - "golang.org/x/sync/errgroup" -) - -// BatchNewLeafNodeData is a struct that contains the data needed to create a new leaf node. -type BatchNewLeafNodeData struct { - Stem []byte - Values map[byte][]byte -} - -// BatchNewLeafNode creates a new leaf node from the given data. It optimizes LeafNode creation -// by batching expensive cryptography operations. It returns the LeafNodes sorted by stem. -func BatchNewLeafNode(nodesValues []BatchNewLeafNodeData) ([]LeafNode, error) { - cfg := GetConfig() - ret := make([]LeafNode, len(nodesValues)) - - numBatches := runtime.NumCPU() - batchSize := len(nodesValues) / numBatches - - group, _ := errgroup.WithContext(context.Background()) - for i := 0; i < numBatches; i++ { - start := i * batchSize - end := (i + 1) * batchSize - if i == numBatches-1 { - end = len(nodesValues) - } - - work := func(ret []LeafNode, nodesValues []BatchNewLeafNodeData) func() error { - return func() error { - c1c2points := make([]*Point, 2*len(nodesValues)) - c1c2frs := make([]*Fr, 2*len(nodesValues)) - for i, nv := range nodesValues { - valsslice := make([][]byte, NodeWidth) - for idx := range nv.Values { - valsslice[idx] = nv.Values[idx] - } - - var leaf *LeafNode - leaf, err := NewLeafNode(nv.Stem, valsslice) - if err != nil { - return err - } - ret[i] = *leaf - - c1c2points[2*i], c1c2points[2*i+1] = ret[i].c1, ret[i].c2 - c1c2frs[2*i], c1c2frs[2*i+1] = new(Fr), new(Fr) - } - - toFrMultiple(c1c2frs, c1c2points) - - var poly [NodeWidth]Fr - poly[0].SetUint64(1) - for i, nv := range nodesValues { - if err := StemFromBytes(&poly[1], nv.Stem); err != nil { - return err - } - poly[2] = *c1c2frs[2*i] - poly[3] = *c1c2frs[2*i+1] - - ret[i].commitment = cfg.CommitToPoly(poly[:], 252) - } - return nil - } - } - group.Go(work(ret[start:end], nodesValues[start:end])) - } - if err := group.Wait(); err != nil { - return nil, fmt.Errorf("creating leaf node: %s", err) - } - - sort.Slice(ret, func(i, j int) bool { - return bytes.Compare(ret[i].stem, ret[j].stem) < 0 - }) - - return ret, nil -} - -// firstDiffByteIdx will return the first index in which the two stems differ. -// Both stems *must* be different. -func firstDiffByteIdx(stem1 []byte, stem2 []byte) int { - for i := range stem1 { - if stem1[i] != stem2[i] { - return i - } - } - panic("stems are equal") -} - -func (n *InternalNode) InsertMigratedLeaves(leaves []LeafNode, resolver NodeResolverFn) error { - for i := range leaves { - ln := leaves[i] - parent := n - - // Look for the appropriate parent for the leaf node. - for { - if hashedNode, ok := parent.children[ln.stem[parent.depth]].(*HashedNode); ok { - serialized, err := resolver(hashedNode.commitment) - if err != nil { - return fmt.Errorf("resolving node %x: %w", hashedNode.commitment, err) - } - resolved, err := ParseNode(serialized, parent.depth+1, hashedNode.commitment) - if err != nil { - return fmt.Errorf("parsing node %x: %w", serialized, err) - } - parent.children[ln.stem[parent.depth]] = resolved - } - - nextParent, ok := parent.children[ln.stem[parent.depth]].(*InternalNode) - if !ok { - break - } - - parent.cowChild(ln.stem[parent.depth]) - parent = nextParent - } - - switch node := parent.children[ln.stem[parent.depth]].(type) { - case Empty: - parent.cowChild(ln.stem[parent.depth]) - parent.children[ln.stem[parent.depth]] = &ln - ln.setDepth(parent.depth + 1) - case *LeafNode: - if bytes.Equal(node.stem, ln.stem) { - // In `ln` we have migrated key/values which should be copied to the leaf - // only if there isn't a value there. If there's a value, we skip it since - // our migrated value is stale. - nonPresentValues := make([][]byte, NodeWidth) - for i := range ln.values { - if node.values[i] == nil { - nonPresentValues[i] = ln.values[i] - } - } - - if err := node.updateMultipleLeaves(nonPresentValues); err != nil { - return fmt.Errorf("updating leaves: %s", err) - } - continue - } - - // Otherwise, we need to create the missing internal nodes depending in the fork point in their stems. - idx := firstDiffByteIdx(node.stem, ln.stem) - // We do a sanity check to make sure that the fork point is not before the current depth. - if byte(idx) <= parent.depth { - return fmt.Errorf("unexpected fork point %d for nodes %x and %x", idx, node.stem, ln.stem) - } - // Create the missing internal nodes. - for i := parent.depth + 1; i <= byte(idx); i++ { - nextParent := newInternalNode(parent.depth + 1).(*InternalNode) - parent.cowChild(ln.stem[parent.depth]) - parent.children[ln.stem[parent.depth]] = nextParent - parent = nextParent - } - // Add old and new leaf node to the latest created parent. - parent.cowChild(node.stem[parent.depth]) - parent.children[node.stem[parent.depth]] = node - node.setDepth(parent.depth + 1) - parent.cowChild(ln.stem[parent.depth]) - parent.children[ln.stem[parent.depth]] = &ln - ln.setDepth(parent.depth + 1) - default: - return fmt.Errorf("unexpected node type %T", node) - } - } - return nil -} diff --git a/encoding_test.go b/encoding_test.go index de666228..abea2f2e 100644 --- a/encoding_test.go +++ b/encoding_test.go @@ -13,10 +13,7 @@ func TestLeafStemLength(t *testing.T) { // Serialize a leaf with no values, but whose stem is 32 bytes. The // serialization should trim the extra byte. toolong := make([]byte, 32) - leaf, err := NewLeafNode(toolong, make([][]byte, NodeWidth)) - if err != nil { - t.Fatal(err) - } + leaf := NewLeafNode(toolong, make([][]byte, NodeWidth)) ser, err := leaf.Serialize() if err != nil { t.Fatal(err) @@ -35,10 +32,7 @@ func TestInvalidNodeEncoding(t *testing.T) { // Test an invalid node type. values := make([][]byte, NodeWidth) values[42] = testValue - ln, err := NewLeafNode(ffx32KeyTest, values) - if err != nil { - t.Fatal(err) - } + ln := NewLeafNode(ffx32KeyTest, values) lnbytes, err := ln.Serialize() if err != nil { t.Fatalf("serializing leaf node: %v", err) diff --git a/tree.go b/tree.go index e9b8cae4..980b65bf 100644 --- a/tree.go +++ b/tree.go @@ -30,6 +30,8 @@ import ( "encoding/json" "errors" "fmt" + "runtime" + "sync" "github.com/crate-crypto/go-ipa/banderwagon" ) @@ -200,6 +202,7 @@ func (n *InternalNode) toExportable() *ExportableInternalNode { case *InternalNode: exportable.Children[i] = child.toExportable() case *LeafNode: + child.Commit() exportable.Children[i] = &ExportableLeafNode{ Stem: child.stem, Values: child.values, @@ -247,57 +250,16 @@ func NewStatelessInternal(depth byte, comm *Point) VerkleNode { } // New creates a new leaf node -func NewLeafNode(stem []byte, values [][]byte) (*LeafNode, error) { - cfg := GetConfig() - - // C1. - var c1poly [NodeWidth]Fr - var c1 *Point - count, err := fillSuffixTreePoly(c1poly[:], values[:NodeWidth/2]) - if err != nil { - return nil, err - } - containsEmptyCodeHash := len(c1poly) >= EmptyCodeHashSecondHalfIdx && - c1poly[EmptyCodeHashFirstHalfIdx].Equal(&EmptyCodeHashFirstHalfValue) && - c1poly[EmptyCodeHashSecondHalfIdx].Equal(&EmptyCodeHashSecondHalfValue) - if containsEmptyCodeHash { - // Clear out values of the cached point. - c1poly[EmptyCodeHashFirstHalfIdx] = FrZero - c1poly[EmptyCodeHashSecondHalfIdx] = FrZero - // Calculate the remaining part of c1 and add to the base value. - partialc1 := cfg.CommitToPoly(c1poly[:], NodeWidth-count-2) - c1 = new(Point) - c1.Add(&EmptyCodeHashPoint, partialc1) - } else { - c1 = cfg.CommitToPoly(c1poly[:], NodeWidth-count) - } - - // C2. - var c2poly [NodeWidth]Fr - count, err = fillSuffixTreePoly(c2poly[:], values[NodeWidth/2:]) - if err != nil { - return nil, err - } - c2 := cfg.CommitToPoly(c2poly[:], NodeWidth-count) - - // Root commitment preparation for calculation. - stem = stem[:StemSize] // enforce a 31-byte length - var poly [NodeWidth]Fr - poly[0].SetUint64(1) - if err := StemFromBytes(&poly[1], stem); err != nil { - return nil, err - } - toFrMultiple([]*Fr{&poly[2], &poly[3]}, []*Point{c1, c2}) - +func NewLeafNode(stem []byte, values [][]byte) *LeafNode { return &LeafNode{ // depth will be 0, but the commitment calculation // does not need it, and so it won't be free. values: values, stem: stem, - commitment: cfg.CommitToPoly(poly[:], NodeWidth-4), - c1: c1, - c2: c2, - }, nil + commitment: nil, + c1: nil, + c2: nil, + } } // NewLeafNodeWithNoComms create a leaf node but does compute its @@ -352,11 +314,7 @@ func (n *InternalNode) InsertStem(stem []byte, values [][]byte, resolver NodeRes case UnknownNode: return errMissingNodeInStateless case Empty: - var err error - n.children[nChild], err = NewLeafNode(stem, values) - if err != nil { - return err - } + n.children[nChild] = NewLeafNode(stem, values) n.children[nChild].setDepth(n.depth + 1) case *HashedNode: if resolver == nil { @@ -397,10 +355,7 @@ func (n *InternalNode) InsertStem(stem []byte, values [][]byte, resolver NodeRes // Next word differs, so this was the last level. // Insert it directly into its final slot. - leaf, err := NewLeafNode(stem, values) - if err != nil { - return err - } + leaf := NewLeafNode(stem, values) leaf.setDepth(n.depth + 2) newBranch.cowChild(nextWordInInsertedKey) newBranch.children[nextWordInInsertedKey] = leaf @@ -669,11 +624,33 @@ func (n *InternalNode) fillLevels(levels [][]*InternalNode) { } } +func (n *InternalNode) findNewLeafNodes(newLeaves []*LeafNode) []*LeafNode { + for idx := range n.cow { + child := n.children[idx] + if childInternalNode, ok := child.(*InternalNode); ok && len(childInternalNode.cow) > 0 { + newLeaves = childInternalNode.findNewLeafNodes(newLeaves) + } else if leafNode, ok := child.(*LeafNode); ok { + if leafNode.commitment == nil { + newLeaves = append(newLeaves, leafNode) + } + } + } + return newLeaves +} + func (n *InternalNode) Commit() *Point { if len(n.cow) == 0 { return n.commitment } + // New leaf nodes. + newLeaves := make([]*LeafNode, 0, 64) + newLeaves = n.findNewLeafNodes(newLeaves) + if len(newLeaves) > 0 { + batchCommitLeafNodes(newLeaves) + } + + // Internal nodes. internalNodeLevels := make([][]*InternalNode, StemSize) n.fillLevels(internalNodeLevels) @@ -1047,6 +1024,14 @@ func (n *LeafNode) updateCn(index byte, value []byte, c *Point) error { } func (n *LeafNode) updateLeaf(index byte, value []byte) error { + // If the commitment is nil, it means this is a new leaf. + // We just update the value since the commitment of all new leaves will + // be calculated when calling Commit(). + if n.commitment == nil { + n.values[index] = value + return nil + } + // Update the corresponding C1 or C2 commitment. var c *Point var oldC Point @@ -1074,6 +1059,19 @@ func (n *LeafNode) updateLeaf(index byte, value []byte) error { } func (n *LeafNode) updateMultipleLeaves(values [][]byte) error { + // If the leaf node commitment is nil, it means this is a new leaf. + // We just update the provided value in the right slot, and we're done. + // The commitment will be calculated when the tree calls Commit(). + if n.commitment == nil { + for i, v := range values { + if len(v) != 0 && !bytes.Equal(v, n.values[i]) { + n.values[i] = v + } + } + return nil + } + + // If the n.commitment isn't nil, we do diff updating. var oldC1, oldC2 *Point // We iterate the values, and we update the C1 and/or C2 commitments depending on the index. @@ -1252,6 +1250,10 @@ func (n *LeafNode) Commitment() *Point { } func (n *LeafNode) Commit() *Point { + if n.commitment == nil { + commitLeafNodes([]*LeafNode{n}) + } + return n.commitment } @@ -1450,6 +1452,7 @@ func (n *LeafNode) GetProofItems(keys keylist) (*ProofElements, []byte, [][]byte // Serialize serializes a LeafNode. // The format is: func (n *LeafNode) Serialize() ([]byte, error) { + n.Commit() cBytes := banderwagon.ElementsToBytes([]*banderwagon.Element{n.c1, n.c2}) return n.serializeWithCompressedCommitments(cBytes[0], cBytes[1]), nil } @@ -1680,3 +1683,83 @@ func (n *LeafNode) serializeWithCompressedCommitments(c1Bytes [32]byte, c2Bytes return result } + +func batchCommitLeafNodes(leaves []*LeafNode) { + minBatchSize := 8 + if len(leaves) < minBatchSize { + commitLeafNodes(leaves) + return + } + + batchSize := len(leaves) / runtime.NumCPU() + if batchSize < minBatchSize { + batchSize = minBatchSize + } + + var wg sync.WaitGroup + for start := 0; start < len(leaves); start += batchSize { + end := start + batchSize + if end > len(leaves) { + end = len(leaves) + } + wg.Add(1) + go func(leaves []*LeafNode) { + defer wg.Done() + commitLeafNodes(leaves) + }(leaves[start:end]) + } + wg.Wait() +} + +func commitLeafNodes(leaves []*LeafNode) error { + cfg := GetConfig() + + c1c2points := make([]*Point, 2*len(leaves)) + c1c2frs := make([]*Fr, 2*len(leaves)) + for i, n := range leaves { + // C1. + var c1poly [NodeWidth]Fr + count, err := fillSuffixTreePoly(c1poly[:], n.values[:NodeWidth/2]) + if err != nil { + return fmt.Errorf("fillSuffixTreePoly for c1: %v", err) + } + containsEmptyCodeHash := len(c1poly) >= EmptyCodeHashSecondHalfIdx && + c1poly[EmptyCodeHashFirstHalfIdx].Equal(&EmptyCodeHashFirstHalfValue) && + c1poly[EmptyCodeHashSecondHalfIdx].Equal(&EmptyCodeHashSecondHalfValue) + if containsEmptyCodeHash { + // Clear out values of the cached point. + c1poly[EmptyCodeHashFirstHalfIdx] = FrZero + c1poly[EmptyCodeHashSecondHalfIdx] = FrZero + // Calculate the remaining part of c1 and add to the base value. + partialc1 := cfg.CommitToPoly(c1poly[:], NodeWidth-count-2) + n.c1 = new(Point) + n.c1.Add(&EmptyCodeHashPoint, partialc1) + } else { + n.c1 = cfg.CommitToPoly(c1poly[:], NodeWidth-count) + } + + // C2. + var c2poly [NodeWidth]Fr + count, err = fillSuffixTreePoly(c2poly[:], n.values[NodeWidth/2:]) + if err != nil { + return fmt.Errorf("fillSuffixTreePoly for c2: %v", err) + } + n.c2 = cfg.CommitToPoly(c2poly[:], NodeWidth-count) + + c1c2points[2*i], c1c2points[2*i+1] = n.c1, n.c2 + c1c2frs[2*i], c1c2frs[2*i+1] = new(Fr), new(Fr) + } + + toFrMultiple(c1c2frs, c1c2points) + + var poly [NodeWidth]Fr + poly[0].SetUint64(1) + for i, nv := range leaves { + StemFromBytes(&poly[1], nv.stem) + poly[2] = *c1c2frs[2*i] + poly[3] = *c1c2frs[2*i+1] + + nv.commitment = cfg.CommitToPoly(poly[:], 252) + } + return nil +} diff --git a/tree_test.go b/tree_test.go index 2f9723b1..6ab17630 100644 --- a/tree_test.go +++ b/tree_test.go @@ -770,7 +770,7 @@ func TestInsertIntoHashedNode(t *testing.T) { resolver := func(h []byte) ([]byte, error) { values := make([][]byte, NodeWidth) values[0] = zeroKeyTest - node, _ := NewLeafNode(zeroKeyTest[:31], values) + node := NewLeafNode(zeroKeyTest[:31], values) return node.Serialize() } @@ -783,7 +783,7 @@ func TestInsertIntoHashedNode(t *testing.T) { invalidRLPResolver := func(h []byte) ([]byte, error) { values := make([][]byte, NodeWidth) values[0] = zeroKeyTest - node, _ := NewLeafNode(zeroKeyTest[:31], values) + node := NewLeafNode(zeroKeyTest[:31], values) rlp, _ := node.Serialize() return rlp[:len(rlp)-10], nil @@ -885,6 +885,7 @@ func TestLeafToCommsLessThan16(*testing.T) { func TestGetProofItemsNoPoaIfStemPresent(t *testing.T) { root := New() root.Insert(ffx32KeyTest, zeroKeyTest, nil) + root.Commit() // insert two keys that differ from the inserted stem // by one byte. @@ -1178,7 +1179,7 @@ func BenchmarkEmptyHashCodeCachedPoint(b *testing.B) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - _, _ = NewLeafNode(zeroKeyTest, values) + _ = NewLeafNode(zeroKeyTest, values) } }) } @@ -1194,7 +1195,8 @@ func TestEmptyHashCodeCachedPoint(t *testing.T) { } values := make([][]byte, NodeWidth) values[CodeHashVectorPosition] = emptyHashCode - ln, _ := NewLeafNode(zeroKeyTest, values) + ln := NewLeafNode(zeroKeyTest, values) + ln.Commit() // Compare the result (which used the cached point) with the expected result which was // calculated by a previous version of the library that didn't use a cached point. @@ -1208,18 +1210,17 @@ func TestEmptyHashCodeCachedPoint(t *testing.T) { } } -func TestBatchMigratedKeyValues(t *testing.T) { +func TestInsertNewLeaves(t *testing.T) { _ = GetConfig() for _, treeInitialKeyValCount := range []int{0, 500, 1_000, 2_000, 5_000} { fmt.Printf("Assuming %d key/values touched by block execution:\n", treeInitialKeyValCount) for _, migrationKeyValueCount := range []int{1_000, 2_000, 5_000, 8_000} { - iterations := 5 - var batchedDuration, unbatchedDuration time.Duration + iterations := 10 + var unbatchedDuration time.Duration for i := 0; i < iterations; i++ { runtime.GC() - // ***Insert the key pairs 'naively' *** rand := mRand.New(mRand.NewSource(42)) //skipcq: GSC-G404 tree := genRandomTree(rand, treeInitialKeyValCount) randomKeyValues := genRandomKeyValues(rand, migrationKeyValueCount) @@ -1230,63 +1231,17 @@ func TestBatchMigratedKeyValues(t *testing.T) { t.Fatalf("failed to insert key: %v", err) } } - unbatchedRoot := tree.Commit().Bytes() + tree.Commit() if _, err := tree.(*InternalNode).BatchSerialize(); err != nil { t.Fatalf("failed to serialize unbatched tree: %v", err) } unbatchedDuration += time.Since(now) - - // ***Insert the key pairs with optimized strategy & methods*** - rand = mRand.New(mRand.NewSource(42)) //skipcq: GSC-G404 - tree = genRandomTree(rand, treeInitialKeyValCount) - randomKeyValues = genRandomKeyValues(rand, migrationKeyValueCount) - - now = time.Now() - // Create LeafNodes in batch mode. - nodeValues := make([]BatchNewLeafNodeData, 0, len(randomKeyValues)) - curr := BatchNewLeafNodeData{ - Stem: randomKeyValues[0].key[:StemSize], - Values: map[byte][]byte{randomKeyValues[0].key[StemSize]: randomKeyValues[0].value}, - } - for _, kv := range randomKeyValues[1:] { - if bytes.Equal(curr.Stem, kv.key[:StemSize]) { - curr.Values[kv.key[StemSize]] = kv.value - continue - } - nodeValues = append(nodeValues, curr) - curr = BatchNewLeafNodeData{ - Stem: kv.key[:StemSize], - Values: map[byte][]byte{kv.key[StemSize]: kv.value}, - } - } - // Append last remaining node. - nodeValues = append(nodeValues, curr) - - // Create all leaves in batch mode so we can optimize cryptography operations. - newLeaves, err := BatchNewLeafNode(nodeValues) - if err != nil { - t.Fatalf("failed to batch create leaf nodes: %v", err) - } - if err = tree.(*InternalNode).InsertMigratedLeaves(newLeaves, nil); err != nil { - t.Fatalf("failed to insert key: %v", err) - } - - batchedRoot := tree.Commit().Bytes() - if _, err := tree.(*InternalNode).BatchSerialize(); err != nil { - t.Fatalf("failed to serialize batched tree: %v", err) - } - batchedDuration += time.Since(now) - - if unbatchedRoot != batchedRoot { - t.Fatalf("expected %x, got %x", unbatchedRoot, batchedRoot) - } } - fmt.Printf("\tIf %d extra key-values are migrated: unbatched %dms, batched %dms, %.02fx\n", migrationKeyValueCount, (unbatchedDuration / time.Duration(iterations)).Milliseconds(), (batchedDuration / time.Duration(iterations)).Milliseconds(), float64(unbatchedDuration.Milliseconds())/float64(batchedDuration.Milliseconds())) + fmt.Printf("\tIf %d extra key-values are migrated: unbatched %dms\n", migrationKeyValueCount, (unbatchedDuration / time.Duration(iterations)).Milliseconds()) } } } - func genRandomTree(rand *mRand.Rand, keyValueCount int) VerkleNode { tree := New() for _, kv := range genRandomKeyValues(rand, keyValueCount) { @@ -1313,7 +1268,7 @@ func genRandomKeyValues(rand *mRand.Rand, count int) []keyValue { return ret } -func BenchmarkBatchLeavesInsert(b *testing.B) { +func BenchmarkNewLeavesInsert(b *testing.B) { treeInitialKeyValCount := 1_000 migrationKeyValueCount := 5_000 @@ -1329,34 +1284,10 @@ func BenchmarkBatchLeavesInsert(b *testing.B) { b.StartTimer() // Create LeafNodes in batch mode. - nodeValues := make([]BatchNewLeafNodeData, 0, len(randomKeyValues)) - curr := BatchNewLeafNodeData{ - Stem: randomKeyValues[0].key[:StemSize], - Values: map[byte][]byte{randomKeyValues[0].key[StemSize]: randomKeyValues[0].value}, - } for _, kv := range randomKeyValues[1:] { - if bytes.Equal(curr.Stem, kv.key[:StemSize]) { - curr.Values[kv.key[StemSize]] = kv.value - continue - } - nodeValues = append(nodeValues, curr) - curr = BatchNewLeafNodeData{ - Stem: kv.key[:StemSize], - Values: map[byte][]byte{kv.key[StemSize]: kv.value}, - } + tree.Insert(kv.key, kv.value, nil) } - // Append last remaining node. - nodeValues = append(nodeValues, curr) - - // Create all leaves in batch mode so we can optimize cryptography operations. - newLeaves, err := BatchNewLeafNode(nodeValues) - if err != nil { - b.Fatalf("failed to batch-create leaf node: %v", err) - } - if err := tree.(*InternalNode).InsertMigratedLeaves(newLeaves, nil); err != nil { - b.Fatalf("failed to insert key: %v", err) - } - + tree.Commit() if _, err := tree.(*InternalNode).BatchSerialize(); err != nil { b.Fatalf("failed to serialize batched tree: %v", err) } @@ -1394,7 +1325,7 @@ func TestLeafNodeInsert(t *testing.T) { values := make([][]byte, NodeWidth) valIdx := 42 values[valIdx] = testValue - ln, _ := NewLeafNode(ffx32KeyTest[:StemSize], values) + ln := NewLeafNode(ffx32KeyTest[:StemSize], values) // Check we get the value correctly via Get(...). getValue, err := ln.Get(append(ffx32KeyTest[:StemSize], byte(valIdx)), nil) From 40168278276f2000219c3c3472ccfeb1685765bf Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 7 Jun 2023 20:26:23 +0200 Subject: [PATCH 2/8] use the github runner instead --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index ea959b8b..f2b67475 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -35,7 +35,7 @@ jobs: run: go vet test: - runs-on: self-hosted + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Set up Go From b40ac27ab0e2a317c7c7091964393efeabd2114a Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 8 Jun 2023 12:27:33 +0200 Subject: [PATCH 3/8] add precomp download to github runner step --- .github/workflows/go.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index f2b67475..df7c967c 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -35,12 +35,14 @@ jobs: run: go vet test: - runs-on: ubuntu-latest + runs-on: self-hosted steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Go uses: actions/setup-go@v2 with: go-version: 1.18 + - name: Download precomp file + run: wget https://github.com/gballet/go-verkle/releases/download/banderwagonv3/precomp && ls -al - name: Test run: go test -v -race ./... From fe8e5e7b2a5eee2e6b266d50317859ad73f13bdf Mon Sep 17 00:00:00 2001 From: parithosh Date: Thu, 8 Jun 2023 15:35:16 +0200 Subject: [PATCH 4/8] downgrading checkout --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index df7c967c..e472bdfa 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -37,7 +37,7 @@ jobs: test: runs-on: self-hosted steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 - name: Set up Go uses: actions/setup-go@v2 with: From 6361ea860cf91cf75b5fba99dc6a125a3c776e80 Mon Sep 17 00:00:00 2001 From: parithosh Date: Thu, 8 Jun 2023 15:47:54 +0200 Subject: [PATCH 5/8] switching to go version --- .github/workflows/go.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index e472bdfa..d4e56d91 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -15,7 +15,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.18 + go-version: 1.20.5 - name: Build run: go build -v ./... @@ -26,7 +26,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.18 + go-version: 1.20.5 - name: Download golangci-lint run: wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s latest - name: Lint @@ -41,7 +41,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.18 + go-version: 1.20.5 - name: Download precomp file run: wget https://github.com/gballet/go-verkle/releases/download/banderwagonv3/precomp && ls -al - name: Test From f240a876796928c9af94ba28a3f89d774dffd0f5 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Fri, 9 Jun 2023 08:27:06 +0200 Subject: [PATCH 6/8] fix one test --- tree.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tree.go b/tree.go index 980b65bf..0e12db03 100644 --- a/tree.go +++ b/tree.go @@ -255,7 +255,7 @@ func NewLeafNode(stem []byte, values [][]byte) *LeafNode { // depth will be 0, but the commitment calculation // does not need it, and so it won't be free. values: values, - stem: stem, + stem: stem[:StemSize], commitment: nil, c1: nil, c2: nil, From c35845170ab0b5d0d27986ee8af5100d46dd4044 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Fri, 9 Jun 2023 09:26:28 -0300 Subject: [PATCH 7/8] tree: fix test Signed-off-by: Ignacio Hagopian --- tree_test.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tree_test.go b/tree_test.go index 6ab17630..3a024304 100644 --- a/tree_test.go +++ b/tree_test.go @@ -1322,13 +1322,18 @@ func TestManipulateChildren(t *testing.T) { } func TestLeafNodeInsert(t *testing.T) { - values := make([][]byte, NodeWidth) valIdx := 42 - values[valIdx] = testValue - ln := NewLeafNode(ffx32KeyTest[:StemSize], values) + ffx31plus42 := append(ffx32KeyTest[:StemSize], byte(valIdx)) + + tree := New() + if err := tree.Insert(ffx31plus42, testValue, nil); err != nil { + t.Fatalf("failed to insert key: %v", err) + } + tree.Commit() + ln := tree.(*InternalNode).Children()[ffx32KeyTest[0]].(*LeafNode) // Check we get the value correctly via Get(...). - getValue, err := ln.Get(append(ffx32KeyTest[:StemSize], byte(valIdx)), nil) + getValue, err := ln.Get(ffx31plus42, nil) if err != nil { t.Fatalf("failed to get leaf node key/value: %v", err) } From 1dd8d331ccb51690574ee673dec4515f3364e41a Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Fri, 9 Jun 2023 09:32:22 -0300 Subject: [PATCH 8/8] make linter happy with a false positive Signed-off-by: Ignacio Hagopian --- tree_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tree_test.go b/tree_test.go index 3a024304..13a6cb9d 100644 --- a/tree_test.go +++ b/tree_test.go @@ -1323,7 +1323,9 @@ func TestManipulateChildren(t *testing.T) { func TestLeafNodeInsert(t *testing.T) { valIdx := 42 - ffx31plus42 := append(ffx32KeyTest[:StemSize], byte(valIdx)) + ffx31plus42 := make([]byte, 32) + copy(ffx31plus42, ffx32KeyTest[:StemSize]) + ffx31plus42[StemSize] = byte(valIdx) tree := New() if err := tree.Insert(ffx31plus42, testValue, nil); err != nil {