From 6465478056039b2252ebf4e5f74d0c1406afb7ff Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Fri, 3 Nov 2023 13:43:55 -0300 Subject: [PATCH] Proof of absence border case test (#413) * proof: add proof of absence border case tests Signed-off-by: Ignacio Hagopian * Proof of absence and presence for the same path with nil proof of presence value (#414) * keep the extension statuses of absent stems Signed-off-by: Ignacio Hagopian * more improvements and tests Signed-off-by: Ignacio Hagopian * fix & extra test case Signed-off-by: Ignacio Hagopian * make steams and extStatuses be 1:1 Signed-off-by: Ignacio Hagopian * cleanup Signed-off-by: Ignacio Hagopian --------- Signed-off-by: Ignacio Hagopian --------- Signed-off-by: Ignacio Hagopian --- proof_ipa.go | 101 ++++++++++++++++++--------------------- proof_test.go | 129 ++++++++++++++++++++++++++++++++++++++++++++++++++ tree.go | 31 +++++++----- tree_test.go | 4 +- 4 files changed, 196 insertions(+), 69 deletions(-) diff --git a/proof_ipa.go b/proof_ipa.go index c244c550..091922cf 100644 --- a/proof_ipa.go +++ b/proof_ipa.go @@ -398,8 +398,9 @@ func PreStateTreeFromProof(proof *Proof, rootC *Point) (VerkleNode, error) { // stems = append(stems, k[:31]) } } - stemIndex := 0 - + if len(stems) != len(proof.ExtStatus) { + return nil, fmt.Errorf("invalid number of stems and extension statuses: %d != %d", len(stems), len(proof.ExtStatus)) + } var ( info = map[string]stemInfo{} paths [][]byte @@ -412,81 +413,73 @@ func PreStateTreeFromProof(proof *Proof, rootC *Point) (VerkleNode, error) { // return nil, fmt.Errorf("proof of absence stems are not sorted") } - // assign one or more stem to each stem info + // We build a cache of paths that have a presence extension status. + pathsWithExtPresent := map[string]struct{}{} + i := 0 for _, es := range proof.ExtStatus { - depth := es >> 3 - path := stems[stemIndex][:depth] + if es&3 == extStatusPresent { + pathsWithExtPresent[string(stems[i][:es>>3])] = struct{}{} + } + i++ + } + + // assign one or more stem to each stem info + for i, es := range proof.ExtStatus { si := stemInfo{ - depth: depth, + depth: es >> 3, stemType: es & 3, } + path := stems[i][:si.depth] switch si.stemType { case extStatusAbsentEmpty: // All keys that are part of a proof of absence, must contain empty // prestate values. If that isn't the case, the proof is invalid. - for i, k := range proof.Keys { // TODO: DoS risk, use map or binary search. - if bytes.HasPrefix(k, path) { - if proof.PreValues[i] != nil { - return nil, fmt.Errorf("proof of absence (empty) stem %x has a value", si.stem) - } + for j := range proof.Keys { // TODO: DoS risk, use map or binary search. + if bytes.HasPrefix(proof.Keys[j], stems[i]) && proof.PreValues[j] != nil { + return nil, fmt.Errorf("proof of absence (empty) stem %x has a value", si.stem) } } case extStatusAbsentOther: - si.stem = poas[0] - poas = poas[1:] // All keys that are part of a proof of absence, must contain empty // prestate values. If that isn't the case, the proof is invalid. - for i, k := range proof.Keys { // TODO: DoS risk, use map or binary search. - if bytes.HasPrefix(k, si.stem) { - if proof.PreValues[i] != nil { - return nil, fmt.Errorf("proof of absence (other) stem %x has a value", si.stem) - } + for j := range proof.Keys { // TODO: DoS risk, use map or binary search. + if bytes.HasPrefix(proof.Keys[j], stems[i]) && proof.PreValues[j] != nil { + return nil, fmt.Errorf("proof of absence (other) stem %x has a value", si.stem) } } - default: - // the first stem could be missing (e.g. the second stem in the - // group is the one that is present. Compare each key to the first - // stem, along the length of the path only. - stemPath := stems[stemIndex][:len(path)] + + // For this absent path, we must first check if this path contains a proof of presence. + // If that is the case, we don't have to do anything since the corresponding leaf will be + // constructed by that extension status (already processed or to be processed). + // In other case, we should get the stem from the list of proof of absence stems. + if _, ok := pathsWithExtPresent[string(path)]; ok { + continue + } + + // Note that this path doesn't have proof of presence (previous if check above), but + // it can have multiple proof of absence. If a previous proof of absence had already + // created the stemInfo for this path, we don't have to do anything. + if _, ok := info[string(path)]; ok { + continue + } + + si.stem = poas[0] + poas = poas[1:] + case extStatusPresent: si.values = map[byte][]byte{} - for i, k := range proof.Keys { // TODO: DoS risk, use map or binary search. - if bytes.Equal(k[:len(path)], stemPath) && proof.PreValues[i] != nil { - si.values[k[31]] = proof.PreValues[i] + si.stem = stems[i] + for j, k := range proof.Keys { // TODO: DoS risk, use map or binary search. + if bytes.Equal(k[:31], si.stem) { + si.values[k[31]] = proof.PreValues[j] si.has_c1 = si.has_c1 || (k[31] < 128) si.has_c2 = si.has_c2 || (k[31] >= 128) - // This key has values, its stem is the one that - // is present. - if si.stem == nil { - si.stem = k[:31] - continue - } - // Any other key with values must have the same - // same previously detected stem. If that isn't the case, - // the proof is invalid. - if !bytes.Equal(si.stem, k[:31]) { - return nil, fmt.Errorf("multiple keys with values found for stem %x", k[:31]) - } } } - // For a proof of presence, we must always have detected a stem. - // If that isn't the case, the proof is invalid. - if si.stem == nil { - return nil, fmt.Errorf("no stem found for path %x", path) - } + default: + return nil, fmt.Errorf("invalid extension status: %d", si.stemType) } info[string(path)] = si paths = append(paths, path) - - // Skip over all the stems that share the same path - // to the extension tree. This happens e.g. if two - // stems have the same path, but one is a proof of - // absence and the other one is present. - stemIndex++ - for ; stemIndex < len(stems); stemIndex++ { - if !bytes.Equal(stems[stemIndex][:depth], path) { - break - } - } } if len(poas) != 0 { diff --git a/proof_test.go b/proof_test.go index 8a5d8b81..5685275c 100644 --- a/proof_test.go +++ b/proof_test.go @@ -1016,6 +1016,47 @@ func TestProofOfAbsenceBorderCase(t *testing.T) { } } +func TestProofOfAbsenceBorderCaseReversed(t *testing.T) { + root := New() + + key1, _ := hex.DecodeString("0001000000000000000000000000000000000000000000000000000000000001") + key2, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000001") + + // Insert an arbitrary value at key 0000000000000000000000000000000000000000000000000000000000000001 + if err := root.Insert(key1, fourtyKeyTest, nil); err != nil { + t.Fatalf("could not insert key: %v", err) + } + + // Generate a proof for the following keys: + // - key1, which is present. + // - key2, which isn't present. + // Note that all three keys will land on the same leaf value. + proof, _, _, _, _ := MakeVerkleMultiProof(root, nil, keylist{key1, key2}, nil) + + serialized, statediff, err := SerializeProof(proof) + if err != nil { + t.Fatalf("could not serialize proof: %v", err) + } + + dproof, err := DeserializeProof(serialized, statediff) + if err != nil { + t.Fatalf("error deserializing proof: %v", err) + } + + droot, err := PreStateTreeFromProof(dproof, root.Commit()) + if err != nil { + t.Fatal(err) + } + + if !droot.Commit().Equal(root.Commit()) { + t.Fatal("differing root commitments") + } + + if !droot.(*InternalNode).children[0].Commit().Equal(root.(*InternalNode).children[0].Commit()) { + t.Fatal("differing commitment for child #0") + } +} + func TestGenerateProofWithOnlyAbsentKeys(t *testing.T) { t.Parallel() @@ -1088,3 +1129,91 @@ func TestGenerateProofWithOnlyAbsentKeys(t *testing.T) { } } } + +func TestProofOfPresenceWithEmptyValue(t *testing.T) { + root := New() + + key1, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000001") + + // Insert an arbitrary value at key 0000000000000000000000000000000000000000000000000000000000000001 + if err := root.Insert(key1, fourtyKeyTest, nil); err != nil { + t.Fatalf("could not insert key: %v", err) + } + + key2, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000002") + proof, _, _, _, _ := MakeVerkleMultiProof(root, nil, keylist{key2}, nil) + + serialized, statediff, err := SerializeProof(proof) + if err != nil { + t.Fatalf("could not serialize proof: %v", err) + } + + dproof, err := DeserializeProof(serialized, statediff) + if err != nil { + t.Fatalf("error deserializing proof: %v", err) + } + + droot, err := PreStateTreeFromProof(dproof, root.Commit()) + if err != nil { + t.Fatal(err) + } + + if !droot.Commit().Equal(root.Commit()) { + t.Fatal("differing root commitments") + } + + if !droot.(*InternalNode).children[0].Commit().Equal(root.(*InternalNode).children[0].Commit()) { + t.Fatal("differing commitment for child #0") + } +} + +func TestDoubleProofOfAbsence(t *testing.T) { + root := New() + + // Insert some keys. + key11, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000001") + key12, _ := hex.DecodeString("0003000000000000000000000000000000000000000000000000000000000001") + + if err := root.Insert(key11, fourtyKeyTest, nil); err != nil { + t.Fatalf("could not insert key: %v", err) + } + if err := root.Insert(key12, fourtyKeyTest, nil); err != nil { + t.Fatalf("could not insert key: %v", err) + } + + // Try to prove to different stems that end up in the same LeafNode without any other proof of presence + // in that leaf node. i.e: two proof of absence in the same leaf node with no proof of presence. + key2, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000100") + key3, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000200") + proof, _, _, _, _ := MakeVerkleMultiProof(root, nil, keylist{key2, key3}, nil) + + serialized, statediff, err := SerializeProof(proof) + if err != nil { + t.Fatalf("could not serialize proof: %v", err) + } + + dproof, err := DeserializeProof(serialized, statediff) + if err != nil { + t.Fatalf("error deserializing proof: %v", err) + } + + droot, err := PreStateTreeFromProof(dproof, root.Commit()) + if err != nil { + t.Fatal(err) + } + + if !droot.Commit().Equal(root.Commit()) { + t.Fatal("differing root commitments") + } + + // Depite we have two proof of absences for different steams, we should only have one + // stem in `others`. i.e: we only need one for both steams. + if len(proof.PoaStems) != 1 { + t.Fatalf("invalid number of proof-of-absence stems: %d", len(proof.PoaStems)) + } + + // We need one extension status for each stem. + if len(proof.ExtStatus) != 2 { + t.Fatalf("invalid number of proof-of-absence stems: %d", len(proof.PoaStems)) + } +} diff --git a/tree.go b/tree.go index ece16366..88c172b6 100644 --- a/tree.go +++ b/tree.go @@ -1464,34 +1464,39 @@ func (n *LeafNode) GetProofItems(keys keylist, _ NodeResolverFn) (*ProofElements pe.Fis = append(pe.Fis, poly[:]) } + addedAbsentStems := map[string]struct{}{} + // Second pass: add the cn-level elements for _, key := range keys { pe.ByPath[string(key[:n.depth])] = n.commitment // Proof of absence: case of a differing stem. - // Add an unopened stem-level node. if !equalPaths(n.stem, key) { - // Corner case: don't add the poa stem if it's - // already present as a proof-of-absence for a - // different key, or for the same key (case of - // multiple missing keys being absent). - // The list of extension statuses has to be of - // length 1 at this level, so skip otherwise. + // If this is the first extension status added for this path, + // add the proof of absence stem (only once). If later we detect a proof of + // presence, we'll clear the list since that proof of presence + // will be enough to provide the stem. if len(esses) == 0 { - esses = append(esses, extStatusAbsentOther|(n.depth<<3)) poass = append(poass, n.stem) } + // Add an extension status absent other for this stem. + // Note we keep a cache to avoid adding the same stem twice (or more) if + // there're multiple keys with the same stem. + if _, ok := addedAbsentStems[string(key[:StemSize])]; !ok { + esses = append(esses, extStatusAbsentOther|(n.depth<<3)) + addedAbsentStems[string(key[:StemSize])] = struct{}{} + } pe.Vals = append(pe.Vals, nil) continue } - // corner case (see previous corner case): if a proof-of-absence - // stem was found, and it now turns out the same stem is used as - // a proof of presence, clear the proof-of-absence list to avoid - // redundancy. + // As mentioned above, if a proof-of-absence stem was found, and + // it now turns out the same stem is used as a proof of presence, + // clear the proof-of-absence list to avoid redundancy. Note that + // we don't delete the extension statuse since that is needed to + // figure out which is the correct stem for this path. if len(poass) > 0 { poass = nil - esses = nil } var ( diff --git a/tree_test.go b/tree_test.go index 42f74b93..c867a427 100644 --- a/tree_test.go +++ b/tree_test.go @@ -1050,8 +1050,8 @@ func TestGetProofItemsNoPoaIfStemPresent(t *testing.T) { if len(poas) != 0 { t.Fatalf("returned %d poas instead of 0", len(poas)) } - if len(esses) != 1 { - t.Fatalf("returned %d extension statuses instead of the expected 1", len(esses)) + if len(esses) != 3 { + t.Fatalf("returned %d extension statuses instead of the expected 3", len(esses)) } }