Favour hashing strings in strmap

inngest · Jan 31, 2024 · 941996a · 941996a
1 parent 61f00a1
commit 941996a
Show file tree

Hide file tree

Showing 18 changed files with 910 additions and 4 deletions.
diff --git a/tree.go → engine.go b/tree.go → engine.go
diff --git a/tree_art.go → engine_art.go b/tree_art.go → engine_art.go
diff --git a/tree_null.go → engine_null.go b/tree_null.go → engine_null.go
diff --git a/engine_stringmap.go b/engine_stringmap.go
@@ -3,8 +3,10 @@ package expr
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"sync"
 
+	"github.com/cespare/xxhash/v2"
 	"github.com/google/cel-go/common/operators"
 	"github.com/ohler55/ojg/jp"
 	"golang.org/x/sync/errgroup"
@@ -83,7 +85,16 @@ func (n *stringLookup) Search(ctx context.Context, variable string, input any) [
 	if !ok {
 		return nil
 	}
-	return n.strings[str]
+	return n.strings[n.hash(str)]
+}
+
+// hash hashes strings quickly via xxhash.  this provides a _somewhat_ collision-free
+// lookup while reducing memory for strings.  note that internally, go maps store the
+// raw key as a string, which uses extra memory.  by compressing all strings via this
+// hash, memory usage grows predictably even with long strings.
+func (n *stringLookup) hash(input string) string {
+	ui := xxhash.Sum64String(input)
+	return strconv.FormatUint(ui, 36)
 }
 
 func (n *stringLookup) Add(ctx context.Context, p ExpressionPart) error {
@@ -93,7 +104,7 @@ func (n *stringLookup) Add(ctx context.Context, p ExpressionPart) error {
 
 	n.lock.Lock()
 	defer n.lock.Unlock()
-	val := p.Predicate.LiteralAsString()
+	val := n.hash(p.Predicate.LiteralAsString())
 
 	n.vars[p.Predicate.Ident] = struct{}{}
 
@@ -114,7 +125,7 @@ func (n *stringLookup) Remove(ctx context.Context, p ExpressionPart) error {
 	n.lock.Lock()
 	defer n.lock.Unlock()
 
-	val := p.Predicate.LiteralAsString()
+	val := n.hash(p.Predicate.LiteralAsString())
 
 	coll, ok := n.strings[val]
 	if !ok {

diff --git a/engine_stringmap_test.go b/engine_stringmap_test.go
@@ -42,7 +42,7 @@ func TestEngineStringmap(t *testing.T) {
 				},
 			})
 			require.NoError(t, err)
-			require.Equal(t, 2, len(s.strings["123"]))
+			require.Equal(t, 2, len(s.strings[s.hash("123")]))
 		})
 	})
 

diff --git a/go.mod b/go.mod
@@ -3,6 +3,7 @@ module github.com/inngest/expr
 go 1.21.0
 
 require (
+	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/google/cel-go v0.18.2
 	github.com/karlseguin/ccache/v2 v2.0.8
 	github.com/ohler55/ojg v1.21.0

diff --git a/go.sum b/go.sum
@@ -1,5 +1,7 @@
 github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
 github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

diff --git a/vendor/github.com/cespare/xxhash/v2/LICENSE.txt b/vendor/github.com/cespare/xxhash/v2/LICENSE.txt
diff --git a/vendor/github.com/cespare/xxhash/v2/README.md b/vendor/github.com/cespare/xxhash/v2/README.md
diff --git a/vendor/github.com/cespare/xxhash/v2/testall.sh b/vendor/github.com/cespare/xxhash/v2/testall.sh
-Original file line number
+Diff line change
@@ Expand Up / @@ -42,7 +42,7 @@ func TestEngineStringmap(t *testing.T) { @@
     				},
     			})
     			require.NoError(t, err)
-    			require.Equal(t, 2, len(s.strings["123"]))
+    			require.Equal(t, 2, len(s.strings[s.hash("123")]))
     		})
     	})
@@ Expand Down @@