diff --git a/proof_ipa.go b/proof_ipa.go index d29f50d8..091922cf 100644 --- a/proof_ipa.go +++ b/proof_ipa.go @@ -387,7 +387,10 @@ type stemInfo struct { // PreStateTreeFromProof builds a stateless prestate tree from the proof. func PreStateTreeFromProof(proof *Proof, rootC *Point) (VerkleNode, error) { // skipcq: GO-R1005 if len(proof.Keys) != len(proof.PreValues) { - return nil, fmt.Errorf("incompatible number of keys and values: %d != %d", len(proof.Keys), len(proof.PreValues)) + return nil, fmt.Errorf("incompatible number of keys and pre-values: %d != %d", len(proof.Keys), len(proof.PreValues)) + } + if len(proof.Keys) != len(proof.PostValues) { + return nil, fmt.Errorf("incompatible number of keys and post-values: %d != %d", len(proof.Keys), len(proof.PostValues)) } stems := make([][]byte, 0, len(proof.Keys)) for _, k := range proof.Keys { @@ -395,8 +398,9 @@ func PreStateTreeFromProof(proof *Proof, rootC *Point) (VerkleNode, error) { // stems = append(stems, k[:31]) } } - stemIndex := 0 - + if len(stems) != len(proof.ExtStatus) { + return nil, fmt.Errorf("invalid number of stems and extension statuses: %d != %d", len(stems), len(proof.ExtStatus)) + } var ( info = map[string]stemInfo{} paths [][]byte @@ -404,49 +408,78 @@ func PreStateTreeFromProof(proof *Proof, rootC *Point) (VerkleNode, error) { // poas = proof.PoaStems ) - // assign one or more stem to each stem info + // The proof of absence stems must be sorted. If that isn't the case, the proof is invalid. + if !sort.IsSorted(bytesSlice(proof.PoaStems)) { + return nil, fmt.Errorf("proof of absence stems are not sorted") + } + + // We build a cache of paths that have a presence extension status. + pathsWithExtPresent := map[string]struct{}{} + i := 0 for _, es := range proof.ExtStatus { - depth := es >> 3 - path := stems[stemIndex][:depth] + if es&3 == extStatusPresent { + pathsWithExtPresent[string(stems[i][:es>>3])] = struct{}{} + } + i++ + } + + // assign one or more stem to each stem info + for i, es := range proof.ExtStatus { si := stemInfo{ - depth: depth, + depth: es >> 3, stemType: es & 3, } + path := stems[i][:si.depth] switch si.stemType { case extStatusAbsentEmpty: + // All keys that are part of a proof of absence, must contain empty + // prestate values. If that isn't the case, the proof is invalid. + for j := range proof.Keys { // TODO: DoS risk, use map or binary search. + if bytes.HasPrefix(proof.Keys[j], stems[i]) && proof.PreValues[j] != nil { + return nil, fmt.Errorf("proof of absence (empty) stem %x has a value", si.stem) + } + } case extStatusAbsentOther: + // All keys that are part of a proof of absence, must contain empty + // prestate values. If that isn't the case, the proof is invalid. + for j := range proof.Keys { // TODO: DoS risk, use map or binary search. + if bytes.HasPrefix(proof.Keys[j], stems[i]) && proof.PreValues[j] != nil { + return nil, fmt.Errorf("proof of absence (other) stem %x has a value", si.stem) + } + } + + // For this absent path, we must first check if this path contains a proof of presence. + // If that is the case, we don't have to do anything since the corresponding leaf will be + // constructed by that extension status (already processed or to be processed). + // In other case, we should get the stem from the list of proof of absence stems. + if _, ok := pathsWithExtPresent[string(path)]; ok { + continue + } + + // Note that this path doesn't have proof of presence (previous if check above), but + // it can have multiple proof of absence. If a previous proof of absence had already + // created the stemInfo for this path, we don't have to do anything. + if _, ok := info[string(path)]; ok { + continue + } + si.stem = poas[0] poas = poas[1:] - default: - // the first stem could be missing (e.g. the second stem in the - // group is the one that is present. Compare each key to the first - // stem, along the length of the path only. - stemPath := stems[stemIndex][:len(path)] + case extStatusPresent: si.values = map[byte][]byte{} - for i, k := range proof.Keys { - if bytes.Equal(k[:len(path)], stemPath) && proof.PreValues[i] != nil { - si.values[k[31]] = proof.PreValues[i] + si.stem = stems[i] + for j, k := range proof.Keys { // TODO: DoS risk, use map or binary search. + if bytes.Equal(k[:31], si.stem) { + si.values[k[31]] = proof.PreValues[j] si.has_c1 = si.has_c1 || (k[31] < 128) si.has_c2 = si.has_c2 || (k[31] >= 128) - // This key has values, its stem is the one that - // is present. - si.stem = k[:31] } } + default: + return nil, fmt.Errorf("invalid extension status: %d", si.stemType) } info[string(path)] = si paths = append(paths, path) - - // Skip over all the stems that share the same path - // to the extension tree. This happens e.g. if two - // stems have the same path, but one is a proof of - // absence and the other one is present. - stemIndex++ - for ; stemIndex < len(stems); stemIndex++ { - if !bytes.Equal(stems[stemIndex][:depth], path) { - break - } - } } if len(poas) != 0 { @@ -513,3 +546,9 @@ func PostStateTreeFromStateDiff(preroot VerkleNode, statediff StateDiff) (Verkle return postroot, nil } + +type bytesSlice [][]byte + +func (x bytesSlice) Len() int { return len(x) } +func (x bytesSlice) Less(i, j int) bool { return bytes.Compare(x[i], x[j]) < 0 } +func (x bytesSlice) Swap(i, j int) { x[i], x[j] = x[j], x[i] } diff --git a/proof_test.go b/proof_test.go index 8a5d8b81..4a2bf454 100644 --- a/proof_test.go +++ b/proof_test.go @@ -550,8 +550,8 @@ func TestProofOfAbsenceNoneMultipleStems(t *testing.T) { t.Fatalf("invalid number of proof-of-absence stems: %d", len(proof.PoaStems)) } - if len(proof.ExtStatus) != 1 { - t.Fatalf("invalid number of none extension statuses: %d ≠ 1", len(proof.ExtStatus)) + if len(proof.ExtStatus) != 2 { + t.Fatalf("invalid number of extension statuses: %d ≠ 2", len(proof.ExtStatus)) } } @@ -1016,6 +1016,47 @@ func TestProofOfAbsenceBorderCase(t *testing.T) { } } +func TestProofOfAbsenceBorderCaseReversed(t *testing.T) { + root := New() + + key1, _ := hex.DecodeString("0001000000000000000000000000000000000000000000000000000000000001") + key2, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000001") + + // Insert an arbitrary value at key 0000000000000000000000000000000000000000000000000000000000000001 + if err := root.Insert(key1, fourtyKeyTest, nil); err != nil { + t.Fatalf("could not insert key: %v", err) + } + + // Generate a proof for the following keys: + // - key1, which is present. + // - key2, which isn't present. + // Note that all three keys will land on the same leaf value. + proof, _, _, _, _ := MakeVerkleMultiProof(root, nil, keylist{key1, key2}, nil) + + serialized, statediff, err := SerializeProof(proof) + if err != nil { + t.Fatalf("could not serialize proof: %v", err) + } + + dproof, err := DeserializeProof(serialized, statediff) + if err != nil { + t.Fatalf("error deserializing proof: %v", err) + } + + droot, err := PreStateTreeFromProof(dproof, root.Commit()) + if err != nil { + t.Fatal(err) + } + + if !droot.Commit().Equal(root.Commit()) { + t.Fatal("differing root commitments") + } + + if !droot.(*InternalNode).children[0].Commit().Equal(root.(*InternalNode).children[0].Commit()) { + t.Fatal("differing commitment for child #0") + } +} + func TestGenerateProofWithOnlyAbsentKeys(t *testing.T) { t.Parallel() @@ -1088,3 +1129,135 @@ func TestGenerateProofWithOnlyAbsentKeys(t *testing.T) { } } } + +func TestProofOfPresenceWithEmptyValue(t *testing.T) { + root := New() + + key1, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000001") + + // Insert an arbitrary value at key 0000000000000000000000000000000000000000000000000000000000000001 + if err := root.Insert(key1, fourtyKeyTest, nil); err != nil { + t.Fatalf("could not insert key: %v", err) + } + + key2, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000002") + proof, _, _, _, _ := MakeVerkleMultiProof(root, nil, keylist{key2}, nil) + + serialized, statediff, err := SerializeProof(proof) + if err != nil { + t.Fatalf("could not serialize proof: %v", err) + } + + dproof, err := DeserializeProof(serialized, statediff) + if err != nil { + t.Fatalf("error deserializing proof: %v", err) + } + + droot, err := PreStateTreeFromProof(dproof, root.Commit()) + if err != nil { + t.Fatal(err) + } + + if !droot.Commit().Equal(root.Commit()) { + t.Fatal("differing root commitments") + } + + if !droot.(*InternalNode).children[0].Commit().Equal(root.(*InternalNode).children[0].Commit()) { + t.Fatal("differing commitment for child #0") + } +} + +func TestDoubleProofOfAbsence(t *testing.T) { + root := New() + + // Insert some keys. + key11, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000001") + key12, _ := hex.DecodeString("0003000000000000000000000000000000000000000000000000000000000001") + + if err := root.Insert(key11, fourtyKeyTest, nil); err != nil { + t.Fatalf("could not insert key: %v", err) + } + if err := root.Insert(key12, fourtyKeyTest, nil); err != nil { + t.Fatalf("could not insert key: %v", err) + } + + // Try to prove to different stems that end up in the same LeafNode without any other proof of presence + // in that leaf node. i.e: two proof of absence in the same leaf node with no proof of presence. + key2, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000100") + key3, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000200") + proof, _, _, _, _ := MakeVerkleMultiProof(root, nil, keylist{key2, key3}, nil) + + serialized, statediff, err := SerializeProof(proof) + if err != nil { + t.Fatalf("could not serialize proof: %v", err) + } + + dproof, err := DeserializeProof(serialized, statediff) + if err != nil { + t.Fatalf("error deserializing proof: %v", err) + } + + droot, err := PreStateTreeFromProof(dproof, root.Commit()) + if err != nil { + t.Fatal(err) + } + + if !droot.Commit().Equal(root.Commit()) { + t.Fatal("differing root commitments") + } + + // Depite we have two proof of absences for different steams, we should only have one + // stem in `others`. i.e: we only need one for both steams. + if len(proof.PoaStems) != 1 { + t.Fatalf("invalid number of proof-of-absence stems: %d", len(proof.PoaStems)) + } + + // We need one extension status for each stem. + if len(proof.ExtStatus) != 2 { + t.Fatalf("invalid number of extension status: %d", len(proof.PoaStems)) + } +} + +func TestProveAbsenceInEmptyHalf(t *testing.T) { + root := New() + + key1, _ := hex.DecodeString("00000000000000000000000000000000000000000000000000000000000000FF") + + if err := root.Insert(key1, fourtyKeyTest, nil); err != nil { + t.Fatalf("could not insert key: %v", err) + } + if err := root.Insert(key1, fourtyKeyTest, nil); err != nil { + t.Fatalf("could not insert key: %v", err) + } + + key2, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000100") + key3, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000000") + proof, _, _, _, _ := MakeVerkleMultiProof(root, nil, keylist{key2, key3}, nil) + + serialized, statediff, err := SerializeProof(proof) + if err != nil { + t.Fatalf("could not serialize proof: %v", err) + } + + dproof, err := DeserializeProof(serialized, statediff) + if err != nil { + t.Fatalf("error deserializing proof: %v", err) + } + + droot, err := PreStateTreeFromProof(dproof, root.Commit()) + if err != nil { + t.Fatal(err) + } + + if !droot.Commit().Equal(root.Commit()) { + t.Fatal("differing root commitments") + } + + if len(proof.PoaStems) != 0 { + t.Fatalf("invalid number of proof-of-absence stems: %d", len(proof.PoaStems)) + } + + if len(proof.ExtStatus) != 2 { + t.Fatalf("invalid number of extension status: %d", len(proof.ExtStatus)) + } +} diff --git a/tree.go b/tree.go index 4efe747d..ba3cbfa1 100644 --- a/tree.go +++ b/tree.go @@ -434,7 +434,7 @@ func (n *InternalNode) InsertValuesAtStem(stem []byte, values [][]byte, resolver // commitments that have not been assigned a node. It returns // the same list, save the commitments that were consumed // during this call. -func (n *InternalNode) CreatePath(path []byte, stemInfo stemInfo, comms []*Point, values [][]byte) ([]*Point, error) { +func (n *InternalNode) CreatePath(path []byte, stemInfo stemInfo, comms []*Point, values [][]byte) ([]*Point, error) { // skipcq: GO-R1005 if len(path) == 0 { return comms, errors.New("invalid path") } @@ -448,6 +448,12 @@ func (n *InternalNode) CreatePath(path []byte, stemInfo stemInfo, comms []*Point // unknown node. n.children[path[0]] = Empty{} case extStatusAbsentOther: + if len(comms) == 0 { + return comms, fmt.Errorf("missing commitment for stem %x", stemInfo.stem) + } + if len(stemInfo.stem) != StemSize { + return comms, fmt.Errorf("invalid stem size %d", len(stemInfo.stem)) + } // insert poa stem newchild := &LeafNode{ commitment: comms[0], @@ -459,6 +465,12 @@ func (n *InternalNode) CreatePath(path []byte, stemInfo stemInfo, comms []*Point n.children[path[0]] = newchild comms = comms[1:] case extStatusPresent: + if len(comms) == 0 { + return comms, fmt.Errorf("missing commitment for stem %x", stemInfo.stem) + } + if len(stemInfo.stem) != StemSize { + return comms, fmt.Errorf("invalid stem size %d", len(stemInfo.stem)) + } // insert stem newchild := &LeafNode{ commitment: comms[0], @@ -469,12 +481,18 @@ func (n *InternalNode) CreatePath(path []byte, stemInfo stemInfo, comms []*Point n.children[path[0]] = newchild comms = comms[1:] if stemInfo.has_c1 { + if len(comms) == 0 { + return comms, fmt.Errorf("missing commitment for stem %x", stemInfo.stem) + } newchild.c1 = comms[0] comms = comms[1:] } else { newchild.c1 = new(Point) } if stemInfo.has_c2 { + if len(comms) == 0 { + return comms, fmt.Errorf("missing commitment for stem %x", stemInfo.stem) + } newchild.c2 = comms[0] comms = comms[1:] } else { @@ -483,6 +501,8 @@ func (n *InternalNode) CreatePath(path []byte, stemInfo stemInfo, comms []*Point for b, value := range stemInfo.values { newchild.values[b] = value } + default: + return comms, fmt.Errorf("invalid stem type %d", stemInfo.stemType) } return comms, nil } @@ -915,12 +935,16 @@ func (n *InternalNode) GetProofItems(keys keylist, resolver NodeResolverFn) (*Pr // commitment, as the value is 0. _, isempty := n.children[childIdx].(Empty) if isempty { - // A question arises here: what if this proof of absence - // corresponds to several stems? Should the ext status be - // repeated as many times? It would be wasteful, so the - // decoding code has to be aware of this corner case. - esses = append(esses, extStatusAbsentEmpty|((n.depth+1)<<3)) + addedStems := map[string]struct{}{} for i := 0; i < len(group); i++ { + if _, ok := addedStems[string(group[i][:StemSize])]; !ok { + // A question arises here: what if this proof of absence + // corresponds to several stems? Should the ext status be + // repeated as many times? It's wasteful, so consider if the + // decoding code can be aware of this corner case. + esses = append(esses, extStatusAbsentEmpty|((n.depth+1)<<3)) + addedStems[string(group[i][:StemSize])] = struct{}{} + } // Append one nil value per key in this missing stem pe.Vals = append(pe.Vals, nil) } @@ -1444,76 +1468,59 @@ func (n *LeafNode) GetProofItems(keys keylist, _ NodeResolverFn) (*ProofElements pe.Fis = append(pe.Fis, poly[:]) } + addedStems := map[string]struct{}{} + // Second pass: add the cn-level elements for _, key := range keys { pe.ByPath[string(key[:n.depth])] = n.commitment // Proof of absence: case of a differing stem. - // Add an unopened stem-level node. if !equalPaths(n.stem, key) { - // Corner case: don't add the poa stem if it's - // already present as a proof-of-absence for a - // different key, or for the same key (case of - // multiple missing keys being absent). - // The list of extension statuses has to be of - // length 1 at this level, so skip otherwise. + // If this is the first extension status added for this path, + // add the proof of absence stem (only once). If later we detect a proof of + // presence, we'll clear the list since that proof of presence + // will be enough to provide the stem. if len(esses) == 0 { - esses = append(esses, extStatusAbsentOther|(n.depth<<3)) poass = append(poass, n.stem) } + // Add an extension status absent other for this stem. + // Note we keep a cache to avoid adding the same stem twice (or more) if + // there're multiple keys with the same stem. + if _, ok := addedStems[string(key[:StemSize])]; !ok { + esses = append(esses, extStatusAbsentOther|(n.depth<<3)) + addedStems[string(key[:StemSize])] = struct{}{} + } pe.Vals = append(pe.Vals, nil) continue } - // corner case (see previous corner case): if a proof-of-absence - // stem was found, and it now turns out the same stem is used as - // a proof of presence, clear the proof-of-absence list to avoid - // redundancy. + // As mentioned above, if a proof-of-absence stem was found, and + // it now turns out the same stem is used as a proof of presence, + // clear the proof-of-absence list to avoid redundancy. Note that + // we don't delete the extension statuses since that is needed to + // figure out which is the correct stem for this path. if len(poass) > 0 { poass = nil - esses = nil } var ( suffix = key[31] suffPoly [NodeWidth]Fr // suffix-level polynomial - count int err error + scomm *Point ) if suffix >= 128 { - count, err = fillSuffixTreePoly(suffPoly[:], n.values[128:]) - if err != nil { - return nil, nil, nil, err + if _, err = fillSuffixTreePoly(suffPoly[:], n.values[128:]); err != nil { + return nil, nil, nil, fmt.Errorf("filling suffix tree poly: %w", err) } + scomm = n.c2 } else { - count, err = fillSuffixTreePoly(suffPoly[:], n.values[:128]) - if err != nil { - return nil, nil, nil, err + if _, err = fillSuffixTreePoly(suffPoly[:], n.values[:128]); err != nil { + return nil, nil, nil, fmt.Errorf("filling suffix tree poly: %w", err) } - } - - // Proof of absence: case of a missing suffix tree. - // - // The suffix tree for this value is missing, i.e. all - // values in the extension-and-suffix tree are grouped - // in the other suffix tree (e.g. C2 if we are looking - // at C1). - if count == 0 { - // TODO(gballet) maintain a count variable at LeafNode level - // so that we know not to build the polynomials in this case, - // as all the information is available before fillSuffixTreePoly - // has to be called, save the count. - esses = append(esses, extStatusAbsentEmpty|(n.depth<<3)) - pe.Vals = append(pe.Vals, nil) - continue - } - - var scomm *Point - if suffix < 128 { scomm = n.c1 - } else { - scomm = n.c2 } + var leaves [2]Fr if n.values[suffix] == nil { // Proof of absence: case of a missing value. @@ -1533,9 +1540,12 @@ func (n *LeafNode) GetProofItems(keys keylist, _ NodeResolverFn) (*ProofElements pe.Yis = append(pe.Yis, &leaves[0], &leaves[1]) pe.Fis = append(pe.Fis, suffPoly[:], suffPoly[:]) pe.Vals = append(pe.Vals, n.values[key[31]]) - if len(esses) == 0 || esses[len(esses)-1] != extStatusPresent|(n.depth<<3) { + + if _, ok := addedStems[string(key[:StemSize])]; !ok { esses = append(esses, extStatusPresent|(n.depth<<3)) + addedStems[string(key[:StemSize])] = struct{}{} } + slotPath := string(key[:n.depth]) + string([]byte{2 + suffix/128}) pe.ByPath[slotPath] = scomm } diff --git a/tree_test.go b/tree_test.go index 42f74b93..c867a427 100644 --- a/tree_test.go +++ b/tree_test.go @@ -1050,8 +1050,8 @@ func TestGetProofItemsNoPoaIfStemPresent(t *testing.T) { if len(poas) != 0 { t.Fatalf("returned %d poas instead of 0", len(poas)) } - if len(esses) != 1 { - t.Fatalf("returned %d extension statuses instead of the expected 1", len(esses)) + if len(esses) != 3 { + t.Fatalf("returned %d extension statuses instead of the expected 3", len(esses)) } }