From 446dbfb7b4add6b812e11ee963515e52aac1a522 Mon Sep 17 00:00:00 2001 From: Tony Holdstock-Brown Date: Mon, 8 Jan 2024 06:21:28 -0800 Subject: [PATCH] Null aggregation (#12) * Parse and check null equality in expressions * Add aggregateable null checks * Make tree lookups concurrent * Add null removal --- expr.go | 165 ++++++++++++++---- expr_test.go | 94 +++++++++- go.mod | 3 +- go.sum | 2 + lift.go | 7 + parser.go | 15 +- tree.go | 3 +- tree_art.go | 80 +++++---- tree_null.go | 93 ++++++++++ vendor/golang.org/x/sync/LICENSE | 27 +++ vendor/golang.org/x/sync/PATENTS | 22 +++ vendor/golang.org/x/sync/errgroup/errgroup.go | 135 ++++++++++++++ vendor/golang.org/x/sync/errgroup/go120.go | 13 ++ .../golang.org/x/sync/errgroup/pre_go120.go | 14 ++ vendor/modules.txt | 3 + 15 files changed, 600 insertions(+), 76 deletions(-) create mode 100644 tree_null.go create mode 100644 vendor/golang.org/x/sync/LICENSE create mode 100644 vendor/golang.org/x/sync/PATENTS create mode 100644 vendor/golang.org/x/sync/errgroup/errgroup.go create mode 100644 vendor/golang.org/x/sync/errgroup/go120.go create mode 100644 vendor/golang.org/x/sync/errgroup/pre_go120.go diff --git a/expr.go b/expr.go index fdbeacb..fa88979 100644 --- a/expr.go +++ b/expr.go @@ -9,6 +9,7 @@ import ( "github.com/google/cel-go/common/operators" "github.com/ohler55/ojg/jp" + "golang.org/x/sync/errgroup" ) var ( @@ -66,10 +67,11 @@ func NewAggregateEvaluator( eval ExpressionEvaluator, ) AggregateEvaluator { return &aggregator{ - eval: eval, - parser: parser, - artIdents: map[string]PredicateTree{}, - lock: &sync.RWMutex{}, + eval: eval, + parser: parser, + artIdents: map[string]PredicateTree{}, + nullLookups: map[string]PredicateTree{}, + lock: &sync.RWMutex{}, } } @@ -94,8 +96,9 @@ type aggregator struct { eval ExpressionEvaluator parser TreeParser - artIdents map[string]PredicateTree - lock *sync.RWMutex + artIdents map[string]PredicateTree + nullLookups map[string]PredicateTree + lock *sync.RWMutex len int32 @@ -155,7 +158,7 @@ func (a *aggregator) Evaluate(ctx context.Context, data map[string]any) ([]Evalu err = errors.Join(err, merr) } - // TODO: Each match here is a potential success. When other trees and operators which are walkable + // Each match here is a potential success. When other trees and operators which are walkable // are added (eg. >= operators on strings), ensure that we find the correct number of matches // for each group ID and then skip evaluating expressions if the number of matches is <= the group // ID's length. @@ -186,17 +189,41 @@ func (a *aggregator) Evaluate(ctx context.Context, data map[string]any) ([]Evalu return result, matched, nil } +// AggregateMatch attempts to match incoming data to all PredicateTrees, resulting in a selection +// of parts of an expression that have matched. func (a *aggregator) AggregateMatch(ctx context.Context, data map[string]any) ([]ExpressionPart, error) { result := []ExpressionPart{} a.lock.RLock() defer a.lock.RUnlock() - // Store the number of times each GroupID has found a match. We need at least - // as many matches as stored in the group ID to consider the match. + // Each match here is a potential success. Ensure that we find the correct number of matches + // for each group ID and then skip evaluating expressions if the number of matches is <= the group + // ID's length. For example, (A && B && C) is a single group ID and must have a count >= 3, + // else we know a required comparason did not match. + // + // Note that having a count >= the group ID value does not guarantee that the expression is valid. counts := map[groupID]int{} // Store all expression parts per group ID for returning. found := map[groupID][]ExpressionPart{} + // protect the above locks with a map. + lock := &sync.Mutex{} + // run lookups concurrently. + eg := errgroup.Group{} + + add := func(all []ExpressionPart) { + // This is called concurrently, so don't mess with maps in goroutines + lock.Lock() + defer lock.Unlock() + + for _, eval := range all { + counts[eval.GroupID] += 1 + if _, ok := found[eval.GroupID]; !ok { + found[eval.GroupID] = []ExpressionPart{} + } + found[eval.GroupID] = append(found[eval.GroupID], eval) + } + } // Iterate through all known variables/idents in the aggregate tree to see if // the data has those keys set. If so, we can immediately evaluate the data with @@ -204,33 +231,54 @@ func (a *aggregator) AggregateMatch(ctx context.Context, data map[string]any) ([ // // TODO: we should iterate through the expression in a top-down order, ensuring that if // any of the top groups fail to match we quit early. - for k, tree := range a.artIdents { - x, err := jp.ParseString(k) - if err != nil { - return nil, err - } - res := x.Get(data) - if len(res) != 1 { - continue - } + for n, item := range a.artIdents { + tree := item + path := n + eg.Go(func() error { + x, err := jp.ParseString(path) + if err != nil { + return err + } + res := x.Get(data) + if len(res) != 1 { + return nil + } - switch cast := res[0].(type) { - case string: - all, ok := tree.Search(ctx, cast) + cast, ok := res[0].(string) if !ok { - continue + // This isn't a string, so we can't compare within the radix tree. + return nil } - for _, eval := range all.Evals { - counts[eval.GroupID] += 1 - if _, ok := found[eval.GroupID]; !ok { - found[eval.GroupID] = []ExpressionPart{} - } - found[eval.GroupID] = append(found[eval.GroupID], eval) + add(tree.Search(ctx, path, cast)) + return nil + }) + } + + // Match on nulls. + for n, item := range a.nullLookups { + tree := item + path := n + eg.Go(func() error { + x, err := jp.ParseString(path) + if err != nil { + return err } - default: - continue - } + + res := x.Get(data) + if len(res) == 0 { + // This isn't present, which matches null in our overloads. Set the + // value to nil. + res = []any{nil} + } + // This matches null, nil (as null), and any non-null items. + add(tree.Search(ctx, path, res[0])) + return nil + }) + } + + if err := eg.Wait(); err != nil { + return nil, err } for k, count := range counts { @@ -393,6 +441,21 @@ func (a *aggregator) iterGroup(ctx context.Context, node *Node, parsed *ParsedEx return true, nil } +func treeType(p Predicate) TreeType { + // switch on type of literal AND operator type. int64/float64 literals require + // btrees, texts require ARTs. + switch p.Literal.(type) { + case string: + return TreeTypeART + case int64, float64: + return TreeTypeBTree + case nil: + return TreeTypeNullMatch + default: + return TreeTypeNone + } +} + // nodeOp represents an op eg. addNode or removeNode type nodeOp func(ctx context.Context, n *Node, parsed *ParsedExpression) error @@ -403,7 +466,7 @@ func (a *aggregator) addNode(ctx context.Context, n *Node, parsed *ParsedExpress defer a.lock.Unlock() // Each node is aggregateable, so add this to the map for fast filtering. - switch n.Predicate.TreeType() { + switch treeType(*n.Predicate) { case TreeTypeART: tree, ok := a.artIdents[n.Predicate.Ident] if !ok { @@ -419,6 +482,21 @@ func (a *aggregator) addNode(ctx context.Context, n *Node, parsed *ParsedExpress } a.artIdents[n.Predicate.Ident] = tree return nil + case TreeTypeNullMatch: + tree, ok := a.nullLookups[n.Predicate.Ident] + if !ok { + tree = newNullMatcher() + } + err := tree.Add(ctx, ExpressionPart{ + GroupID: n.GroupID, + Predicate: *n.Predicate, + Parsed: parsed, + }) + if err != nil { + return err + } + a.nullLookups[n.Predicate.Ident] = tree + return nil } return errTreeUnimplemented } @@ -430,11 +508,11 @@ func (a *aggregator) removeNode(ctx context.Context, n *Node, parsed *ParsedExpr defer a.lock.Unlock() // Each node is aggregateable, so add this to the map for fast filtering. - switch n.Predicate.TreeType() { + switch treeType(*n.Predicate) { case TreeTypeART: tree, ok := a.artIdents[n.Predicate.Ident] if !ok { - tree = newArtTree() + return ErrExpressionPartNotFound } err := tree.Remove(ctx, ExpressionPart{ GroupID: n.GroupID, @@ -446,6 +524,21 @@ func (a *aggregator) removeNode(ctx context.Context, n *Node, parsed *ParsedExpr } a.artIdents[n.Predicate.Ident] = tree return nil + case TreeTypeNullMatch: + tree, ok := a.nullLookups[n.Predicate.Ident] + if !ok { + return ErrExpressionPartNotFound + } + err := tree.Remove(ctx, ExpressionPart{ + GroupID: n.GroupID, + Predicate: *n.Predicate, + Parsed: parsed, + }) + if err != nil { + return err + } + a.nullLookups[n.Predicate.Ident] = tree + return nil } return errTreeUnimplemented } @@ -479,6 +572,10 @@ func isAggregateable(n *Node) bool { case int64, float64: // TODO: Add binary tree matching for ints/floats return false + case nil: + // This is null, which is supported and a simple lookup to check + // if the event's key in question is present and is not nil. + return true default: return false } diff --git a/expr_test.go b/expr_test.go index 641136d..4dcf7ab 100644 --- a/expr_test.go +++ b/expr_test.go @@ -55,7 +55,7 @@ func evaluate(b *testing.B, i int, parser TreeParser) error { return nil } -func TestEvaluate(t *testing.T) { +func TestEvaluate_Strings(t *testing.T) { ctx := context.Background() parser := NewTreeParser(NewCachingParser(newEnv(), nil)) e := NewAggregateEvaluator(parser, testBoolEvaluator) @@ -479,7 +479,99 @@ func TestEmptyExpressions(t *testing.T) { require.Equal(t, 0, e.ConstantLen()) require.Equal(t, 0, e.AggregateableLen()) }) +} + +func TestEvaluate_Null(t *testing.T) { + ctx := context.Background() + parser, err := newParser() + require.NoError(t, err) + + e := NewAggregateEvaluator(parser, testBoolEvaluator) + + notNull := tex(`event.ts != null`, "id-1") + isNull := tex(`event.ts == null`, "id-2") + + t.Run("Adding a `null` check succeeds and is aggregateable", func(t *testing.T) { + ok, err := e.Add(ctx, notNull) + require.NoError(t, err) + require.True(t, ok) + + ok, err = e.Add(ctx, isNull) + require.NoError(t, err) + require.True(t, ok) + + require.Equal(t, 2, e.Len()) + require.Equal(t, 0, e.ConstantLen()) + require.Equal(t, 2, e.AggregateableLen()) + }) + + t.Run("Not null checks succeed", func(t *testing.T) { + // Matching this expr should now fail. + eval, count, err := e.Evaluate(ctx, map[string]any{ + "event": map[string]any{ + "ts": time.Now().UnixMilli(), + }, + }) + require.NoError(t, err) + require.EqualValues(t, 1, len(eval)) + require.EqualValues(t, 1, count) + require.EqualValues(t, notNull, eval[0]) + }) + + t.Run("Is null checks succeed", func(t *testing.T) { + // Matching this expr should work, as "ts" is nil + eval, count, err := e.Evaluate(ctx, map[string]any{ + "event": map[string]any{ + "ts": nil, + }, + }) + require.NoError(t, err) + require.EqualValues(t, 1, len(eval)) + require.EqualValues(t, 1, count) + require.EqualValues(t, isNull, eval[0]) + }) + t.Run("It removes null checks", func(t *testing.T) { + err := e.Remove(ctx, notNull) + require.NoError(t, err) + + require.Equal(t, 1, e.Len()) + require.Equal(t, 0, e.ConstantLen()) + require.Equal(t, 1, e.AggregateableLen()) + + // We should still match on `isNull` + t.Run("Is null checks succeed", func(t *testing.T) { + // Matching this expr should work, as "ts" is nil + eval, count, err := e.Evaluate(ctx, map[string]any{ + "event": map[string]any{ + "ts": nil, + }, + }) + require.NoError(t, err) + require.EqualValues(t, 1, len(eval)) + require.EqualValues(t, 1, count) + require.EqualValues(t, isNull, eval[0]) + }) + + err = e.Remove(ctx, isNull) + require.NoError(t, err) + require.Equal(t, 0, e.Len()) + require.Equal(t, 0, e.ConstantLen()) + require.Equal(t, 0, e.AggregateableLen()) + + // We should no longer match on `isNull` + t.Run("Is null checks succeed", func(t *testing.T) { + // Matching this expr should work, as "ts" is nil + eval, count, err := e.Evaluate(ctx, map[string]any{ + "event": map[string]any{ + "ts": nil, + }, + }) + require.NoError(t, err) + require.EqualValues(t, 0, len(eval)) + require.EqualValues(t, 0, count) + }) + }) } // tex represents a test Evaluable expression diff --git a/go.mod b/go.mod index 33be73f..93632d9 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,8 @@ require ( github.com/ohler55/ojg v1.21.0 github.com/plar/go-adaptive-radix-tree v1.0.5 github.com/stretchr/testify v1.8.4 + golang.org/x/sync v0.6.0 + google.golang.org/protobuf v1.31.0 ) require ( @@ -19,6 +21,5 @@ require ( golang.org/x/text v0.9.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect - google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index f8dc05c..8533854 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,8 @@ github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlV github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/lift.go b/lift.go index 14bde98..15b93f0 100644 --- a/lift.go +++ b/lift.go @@ -36,6 +36,13 @@ type LiftedArgs interface { // liftLiterals lifts quoted literals into variables, allowing us to normalize // expressions to increase cache hit rates. func liftLiterals(expr string) (string, LiftedArgs) { + if strings.Contains(expr, VarPrefix+".") { + // Do not lift an expression twice, else we run the risk of using + // eg. `vars.a` to reference two separate strings, breaking the + // expression. + return expr, nil + } + // TODO: Lift numeric literals out of expressions. lp := liftParser{expr: expr} return lp.lift() diff --git a/parser.go b/parser.go index 2f72233..a2de17f 100644 --- a/parser.go +++ b/parser.go @@ -12,6 +12,7 @@ import ( "github.com/google/cel-go/cel" celast "github.com/google/cel-go/common/ast" "github.com/google/cel-go/common/operators" + "google.golang.org/protobuf/types/known/structpb" ) // TreeParser parses an expression into a tree, with a root node and branches for @@ -337,6 +338,8 @@ func (p Predicate) TreeType() TreeType { return TreeTypeART case int64, float64: return TreeTypeBTree + case nil: + return TreeTypeNullMatch default: return TreeTypeNone } @@ -573,6 +576,12 @@ func callToPredicate(item celast.Expr, negated bool, vars LiftedArgs) *Predicate } } + // If the literal is of type `structpb.NullValue`, replace this with a simple `nil` + // to make nil checks easy. + if _, ok := literal.(structpb.NullValue); ok { + literal = nil + } + if identA != "" && identB != "" { // We're matching two variables together. Check to see whether any // of these idents have variable data being passed in above. @@ -628,9 +637,9 @@ func callToPredicate(item celast.Expr, negated bool, vars LiftedArgs) *Predicate } } - if identA == "" || literal == nil { - return nil - } + // if identA == "" || literal == nil { + // return nil + // } // We always assume that the ident is on the LHS. In the case of comparisons, // we need to switch these and the operator if the literal is on the RHS. This lets diff --git a/tree.go b/tree.go index 97ab389..487b973 100644 --- a/tree.go +++ b/tree.go @@ -11,6 +11,7 @@ const ( TreeTypeART TreeTypeBTree + TreeTypeNullMatch ) // PredicateTree represents a tree which matches a predicate over @@ -21,7 +22,7 @@ const ( type PredicateTree interface { Add(ctx context.Context, p ExpressionPart) error Remove(ctx context.Context, p ExpressionPart) error - Search(ctx context.Context, input any) (*Leaf, bool) + Search(ctx context.Context, variable string, input any) []ExpressionPart } // Leaf represents the leaf within a tree. This stores all expressions diff --git a/tree_art.go b/tree_art.go index 9a2b31e..9ad0cad 100644 --- a/tree_art.go +++ b/tree_art.go @@ -26,27 +26,34 @@ type artTree struct { art.Tree } -func (a *artTree) Search(ctx context.Context, input any) (*Leaf, bool) { - var key art.Key - - switch val := input.(type) { - case art.Key: - key = val - case []byte: - key = val - case string: - key = artKeyFromString(val) +func (a *artTree) Add(ctx context.Context, p ExpressionPart) error { + str, ok := p.Predicate.Literal.(string) + if !ok { + return ErrInvalidType } - if len(key) == 0 { - return nil, false - } + key := artKeyFromString(str) + + // Don't allow multiple gorutines to modify the tree simultaneously. + a.lock.Lock() + defer a.lock.Unlock() val, ok := a.Tree.Search(key) if !ok { - return nil, false + // Insert the ExpressionPart as-is. + a.Insert(key, art.Value(&Leaf{ + Evals: []ExpressionPart{p}, + })) + return nil } - return val.(*Leaf), true + + // Add the expressionpart as an expression matched by the already-existing + // value. Many expressions may match on the same string, eg. a user may set + // up 3 matches for order ID "abc". All 3 matches must be evaluated. + next := val.(*Leaf) + next.Evals = append(next.Evals, p) + a.Insert(key, next) + return nil } func (a *artTree) Remove(ctx context.Context, p ExpressionPart) error { @@ -79,34 +86,35 @@ func (a *artTree) Remove(ctx context.Context, p ExpressionPart) error { return ErrExpressionPartNotFound } -func (a *artTree) Add(ctx context.Context, p ExpressionPart) error { - str, ok := p.Predicate.Literal.(string) - if !ok { - return ErrInvalidType +func (a *artTree) Search(ctx context.Context, variable string, input any) []ExpressionPart { + leaf, ok := a.searchLeaf(ctx, input) + if !ok || leaf == nil { + return nil } + return leaf.Evals +} - key := artKeyFromString(str) +func (a *artTree) searchLeaf(ctx context.Context, input any) (*Leaf, bool) { + var key art.Key - // Don't allow multiple gorutines to modify the tree simultaneously. - a.lock.Lock() - defer a.lock.Unlock() + switch val := input.(type) { + case art.Key: + key = val + case []byte: + key = val + case string: + key = artKeyFromString(val) + } + + if len(key) == 0 { + return nil, false + } val, ok := a.Tree.Search(key) if !ok { - // Insert the ExpressionPart as-is. - a.Insert(key, art.Value(&Leaf{ - Evals: []ExpressionPart{p}, - })) - return nil + return nil, false } - - // Add the expressionpart as an expression matched by the already-existing - // value. Many expressions may match on the same string, eg. a user may set - // up 3 matches for order ID "abc". All 3 matches must be evaluated. - next := val.(*Leaf) - next.Evals = append(next.Evals, p) - a.Insert(key, next) - return nil + return val.(*Leaf), true } func artKeyFromString(str string) art.Key { diff --git a/tree_null.go b/tree_null.go new file mode 100644 index 0000000..46a07d1 --- /dev/null +++ b/tree_null.go @@ -0,0 +1,93 @@ +package expr + +import ( + "context" + "sync" + + "github.com/google/cel-go/common/operators" +) + +// TODO: Rename PredicateTrees as these may not be trees -.- +func newNullMatcher() PredicateTree { + return &nullLookup{ + lock: &sync.RWMutex{}, + null: map[string][]ExpressionPart{}, + not: map[string][]ExpressionPart{}, + } +} + +type nullLookup struct { + lock *sync.RWMutex + null map[string][]ExpressionPart + not map[string][]ExpressionPart +} + +func (n nullLookup) Add(ctx context.Context, p ExpressionPart) error { + n.lock.Lock() + defer n.lock.Unlock() + + varName := p.Predicate.Ident + + // If we're comparing to null ("a" == null), we want the variable + // to be null and should place this in the `null` map. + // + // Any other comparison is a not-null comparison. + if p.Predicate.Operator == operators.Equals { + if _, ok := n.null[varName]; !ok { + n.null[varName] = []ExpressionPart{p} + return nil + } + n.null[varName] = append(n.null[varName], p) + return nil + } + + if _, ok := n.not[varName]; !ok { + n.not[varName] = []ExpressionPart{p} + return nil + } + n.not[varName] = append(n.not[varName], p) + return nil +} + +func (n *nullLookup) Remove(ctx context.Context, p ExpressionPart) error { + n.lock.Lock() + defer n.lock.Unlock() + + coll, ok := n.not[p.Predicate.Ident] + if p.Predicate.Operator == operators.Equals { + coll, ok = n.null[p.Predicate.Ident] + } + + if !ok { + // This could not exist as there's nothing mapping this variable for + // the given event name. + return ErrExpressionPartNotFound + } + + // Remove the expression part from the leaf. + for i, eval := range coll { + if p.Equals(eval) { + coll = append(coll[:i], coll[i+1:]...) + if p.Predicate.Operator == operators.Equals { + n.null[p.Predicate.Ident] = coll + } else { + n.not[p.Predicate.Ident] = coll + } + return nil + } + } + + return ErrExpressionPartNotFound +} + +func (n *nullLookup) Search(ctx context.Context, variable string, input any) []ExpressionPart { + if input == nil { + // The input data is null, so the only items that can match are equality + // comparisons to null. + all := n.null[variable] + return all + } + + all := n.not[variable] + return all +} diff --git a/vendor/golang.org/x/sync/LICENSE b/vendor/golang.org/x/sync/LICENSE new file mode 100644 index 0000000..6a66aea --- /dev/null +++ b/vendor/golang.org/x/sync/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/golang.org/x/sync/PATENTS b/vendor/golang.org/x/sync/PATENTS new file mode 100644 index 0000000..7330990 --- /dev/null +++ b/vendor/golang.org/x/sync/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/vendor/golang.org/x/sync/errgroup/errgroup.go b/vendor/golang.org/x/sync/errgroup/errgroup.go new file mode 100644 index 0000000..948a3ee --- /dev/null +++ b/vendor/golang.org/x/sync/errgroup/errgroup.go @@ -0,0 +1,135 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package errgroup provides synchronization, error propagation, and Context +// cancelation for groups of goroutines working on subtasks of a common task. +// +// [errgroup.Group] is related to [sync.WaitGroup] but adds handling of tasks +// returning errors. +package errgroup + +import ( + "context" + "fmt" + "sync" +) + +type token struct{} + +// A Group is a collection of goroutines working on subtasks that are part of +// the same overall task. +// +// A zero Group is valid, has no limit on the number of active goroutines, +// and does not cancel on error. +type Group struct { + cancel func(error) + + wg sync.WaitGroup + + sem chan token + + errOnce sync.Once + err error +} + +func (g *Group) done() { + if g.sem != nil { + <-g.sem + } + g.wg.Done() +} + +// WithContext returns a new Group and an associated Context derived from ctx. +// +// The derived Context is canceled the first time a function passed to Go +// returns a non-nil error or the first time Wait returns, whichever occurs +// first. +func WithContext(ctx context.Context) (*Group, context.Context) { + ctx, cancel := withCancelCause(ctx) + return &Group{cancel: cancel}, ctx +} + +// Wait blocks until all function calls from the Go method have returned, then +// returns the first non-nil error (if any) from them. +func (g *Group) Wait() error { + g.wg.Wait() + if g.cancel != nil { + g.cancel(g.err) + } + return g.err +} + +// Go calls the given function in a new goroutine. +// It blocks until the new goroutine can be added without the number of +// active goroutines in the group exceeding the configured limit. +// +// The first call to return a non-nil error cancels the group's context, if the +// group was created by calling WithContext. The error will be returned by Wait. +func (g *Group) Go(f func() error) { + if g.sem != nil { + g.sem <- token{} + } + + g.wg.Add(1) + go func() { + defer g.done() + + if err := f(); err != nil { + g.errOnce.Do(func() { + g.err = err + if g.cancel != nil { + g.cancel(g.err) + } + }) + } + }() +} + +// TryGo calls the given function in a new goroutine only if the number of +// active goroutines in the group is currently below the configured limit. +// +// The return value reports whether the goroutine was started. +func (g *Group) TryGo(f func() error) bool { + if g.sem != nil { + select { + case g.sem <- token{}: + // Note: this allows barging iff channels in general allow barging. + default: + return false + } + } + + g.wg.Add(1) + go func() { + defer g.done() + + if err := f(); err != nil { + g.errOnce.Do(func() { + g.err = err + if g.cancel != nil { + g.cancel(g.err) + } + }) + } + }() + return true +} + +// SetLimit limits the number of active goroutines in this group to at most n. +// A negative value indicates no limit. +// +// Any subsequent call to the Go method will block until it can add an active +// goroutine without exceeding the configured limit. +// +// The limit must not be modified while any goroutines in the group are active. +func (g *Group) SetLimit(n int) { + if n < 0 { + g.sem = nil + return + } + if len(g.sem) != 0 { + panic(fmt.Errorf("errgroup: modify limit while %v goroutines in the group are still active", len(g.sem))) + } + g.sem = make(chan token, n) +} diff --git a/vendor/golang.org/x/sync/errgroup/go120.go b/vendor/golang.org/x/sync/errgroup/go120.go new file mode 100644 index 0000000..f93c740 --- /dev/null +++ b/vendor/golang.org/x/sync/errgroup/go120.go @@ -0,0 +1,13 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.20 + +package errgroup + +import "context" + +func withCancelCause(parent context.Context) (context.Context, func(error)) { + return context.WithCancelCause(parent) +} diff --git a/vendor/golang.org/x/sync/errgroup/pre_go120.go b/vendor/golang.org/x/sync/errgroup/pre_go120.go new file mode 100644 index 0000000..88ce334 --- /dev/null +++ b/vendor/golang.org/x/sync/errgroup/pre_go120.go @@ -0,0 +1,14 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !go1.20 + +package errgroup + +import "context" + +func withCancelCause(parent context.Context) (context.Context, func(error)) { + ctx, cancel := context.WithCancel(parent) + return ctx, func(error) { cancel() } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 1397f8c..d791e94 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -52,6 +52,9 @@ github.com/stretchr/testify/require ## explicit; go 1.20 golang.org/x/exp/constraints golang.org/x/exp/slices +# golang.org/x/sync v0.6.0 +## explicit; go 1.18 +golang.org/x/sync/errgroup # golang.org/x/text v0.9.0 ## explicit; go 1.17 golang.org/x/text/transform